From 3f24ff44e09ac692dd949a0b3fdb20f7e8274587 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 21 Apr 2020 11:07:54 +0800 Subject: [PATCH 01/63] [td-169] fix limit offset error for table projection/diff query --- src/query/src/queryExecutor.c | 1324 +++++++++++++++++++-------------- 1 file changed, 765 insertions(+), 559 deletions(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 6ded52d11d..e958a53e87 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -16,17 +16,17 @@ #include "hash.h" #include "hashfunc.h" +#include "qast.h" +#include "qresultBuf.h" +#include "query.h" +#include "queryExecutor.h" +#include "queryLog.h" +#include "queryUtil.h" #include "taosmsg.h" #include "tlosertree.h" #include "tscompression.h" +#include "tsdbMain.h" //todo use TableId instead of STable object #include "ttime.h" -#include "qast.h" -#include "qresultBuf.h" -#include "queryExecutor.h" -#include "queryUtil.h" -#include "query.h" -#include "tsdbMain.h" //todo use TableId instead of STable object -#include "queryLog.h" #define DEFAULT_INTERN_BUF_SIZE 16384L @@ -45,16 +45,15 @@ #define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN) #define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) -#define GET_QINFO_ADDR(x) ((void*)((char *)(x)-offsetof(SQInfo, runtimeEnv))) +#define GET_QINFO_ADDR(x) ((void *)((char *)(x)-offsetof(SQInfo, runtimeEnv))) #define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index) * (step)) -#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC)) +#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) /* get the qinfo struct address from the query struct address */ #define GET_COLUMN_BYTES(query, colidx) \ ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.bytes) -#define GET_COLUMN_TYPE(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.type) +#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.type) typedef struct SPointInterpoSupporter { int32_t numOfCols; @@ -76,11 +75,11 @@ typedef enum { * 2. when all data within queried time window, it is also denoted as query_completed */ QUERY_COMPLETED = 0x4u, - + /* when the result is not completed return to client, this status will be * usually used in case of interval query with interpolation option */ - QUERY_OVER = 0x8u, + QUERY_OVER = 0x8u, } vnodeQueryStatus; enum { @@ -90,19 +89,19 @@ enum { }; typedef struct { - int32_t status; // query status - TSKEY lastKey; // the lastKey value before query executed - STimeWindow w; // whole query time window - STimeWindow current; // current query window - int32_t windowIndex; // index of active time window result for interval query - STSCursor cur; + int32_t status; // query status + TSKEY lastKey; // the lastKey value before query executed + STimeWindow w; // whole query time window + STimeWindow curWindow; // current query window + int32_t windowIndex; // index of active time window result for interval query + STSCursor cur; } SQueryStatusInfo; static void setQueryStatus(SQuery *pQuery, int8_t status); bool isIntervalQuery(SQuery *pQuery) { return pQuery->intervalTime > 0; } -static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* group); -static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); +static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); +static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); @@ -114,9 +113,9 @@ static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); -static void createTableDataInfo(SQInfo* pQInfo); +static void createTableDataInfo(SQInfo *pQInfo); -static int32_t setAdditionalInfo(SQInfo *pQInfo, STable* pTable, STableQueryInfo *pTableQueryInfo); +static int32_t setAdditionalInfo(SQInfo *pQInfo, STable *pTable, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQInfo *pQInfo); bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter *pPointInterpSupporter) { @@ -374,7 +373,7 @@ bool doRevisedResultsByLimit(SQInfo *pQInfo) { if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; assert(pQuery->rec.rows > 0); - + setQueryStatus(pQuery, QUERY_COMPLETED); return true; } @@ -427,21 +426,21 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo */ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SDataStatis **pColStatis) { - SColIndex* pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; + SColIndex *pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; if (TSDB_COL_IS_TAG(pColIndex->flag)) { return false; } - + // query on primary timestamp column, not null value at all if (pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } - + *pColStatis = NULL; if (pStatis != NULL) { *pColStatis = getStatisInfo(pQuery, pStatis, pDataBlockInfo, col); } - + if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { return false; } @@ -865,9 +864,9 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 dataBlock = NULL; } else { /* - * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare stage, - * the remain meter may not have the required column in cache actually. - * So, the validation of required column in cache with the corresponding meter schema is reinforced. + * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare + * stage, the remain meter may not have the required column in cache actually. So, the validation of required + * column in cache with the corresponding meter schema is reinforced. */ if (pDataBlock == NULL) { return NULL; @@ -898,32 +897,31 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * such as count/min/max etc. */ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, - SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, - __block_search_fn_t searchFn, SArray *pDataBlock) { + SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, + __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; SQuery * pQuery = pRuntimeEnv->pQuery; SColumnInfoData *pColInfo = NULL; - TSKEY * primaryKeyCol = NULL; + TSKEY * primaryKeyCol = NULL; if (pDataBlock != NULL) { pColInfo = taosArrayGet(pDataBlock, 0); primaryKeyCol = (TSKEY *)(pColInfo->pData); } - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : pDataBlockInfo->rows - 1; SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; SDataStatis *tpField = NULL; - - bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); + + bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &tpField); char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, - hasNull, &sasArray[k], pRuntimeEnv->scanFlag); + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull, + &sasArray[k], pRuntimeEnv->scanFlag); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -979,7 +977,7 @@ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStati } } } - + tfree(sasArray); } @@ -1131,13 +1129,13 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat bool hasNull = hasNullValue(pQuery, k, pDataBlockInfo, pStatis, &pColStatis); char *dataBlock = getDataBlocks(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, - hasNull, &sasArray[k], pRuntimeEnv->scanFlag); + setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull, + &sasArray[k], pRuntimeEnv->scanFlag); } // set the input column data for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { -// SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + // SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; assert(0); /* * NOTE: here the tbname/tags column cannot reach here, since it will never be a filter column, @@ -1308,17 +1306,17 @@ static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, __block_search_fn_t searchFn, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { - + SDataStatis *pStatis, __block_search_fn_t searchFn, + SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - /*numOfRes = */rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + /*numOfRes = */ rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } - - TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; + + TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; pQuery->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); @@ -1336,7 +1334,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl if (numOfRes > 0 && pQuery->checkBuffer == 1) { assert(numOfRes >= pQuery->rec.rows); pQuery->rec.rows = numOfRes; - + if (numOfRes >= pQuery->rec.threshold) { setQueryStatus(pQuery, QUERY_RESBUF_FULL); } @@ -1359,16 +1357,20 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY } else { pCtx->preAggVals.isSet = false; } - - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0 && (tsCol != NULL)) { - pCtx->ptsList = tsCol; + + pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery)? pQuery->pos : 0; + pCtx->size = QUERY_IS_ASC_QUERY(pQuery)? size - pQuery->pos : pQuery->pos + 1; + + uint32_t status = aAggs[functionId].nStatus; + if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { + pCtx->ptsList = &tsCol[pCtx->startOffset]; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { // last_dist or first_dist function // store the first&last timestamp into the intermediate buffer [1], the true // value may be null but timestamp will never be null - pCtx->ptsList = tsCol; +// pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { /* @@ -1384,15 +1386,12 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY pTWAInfo->EKey = pQuery->window.ekey; } - pCtx->ptsList = tsCol; +// pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[1].pz = param; } - - pCtx->startOffset = 0; - pCtx->size = size; - + #if defined(_DEBUG_VIEW) // int64_t *tsList = (int64_t *)primaryColumnData; // int64_t s = tsList[0]; @@ -1446,7 +1445,7 @@ static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool i } } -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, SColumnModel *pTagsSchema, int16_t order) { +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) { qTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; @@ -1562,7 +1561,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { destroyResultBuf(pRuntimeEnv->pResultBuf); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - + pRuntimeEnv->pTSBuf = tsBufDestory(pRuntimeEnv->pTSBuf); } @@ -1583,9 +1582,7 @@ static bool isQueryKilled(SQInfo *pQInfo) { #endif } -static void setQueryKilled(SQInfo* pQInfo) { - pQInfo->code = TSDB_CODE_QUERY_CANCELLED; -} +static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_QUERY_CANCELLED; } bool isFixedOutputQuery(SQuery *pQuery) { if (pQuery->intervalTime != 0) { @@ -1679,8 +1676,8 @@ static bool needReverseScan(SQuery *pQuery) { } ///////////////////////////////////////////////////////////////////////////////////////////// -void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, - int64_t *realSkey, int64_t *realEkey, STimeWindow *win) { +void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, int64_t *realSkey, + int64_t *realEkey, STimeWindow *win) { assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime); win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->slidingTimeUnit, pQuery->precision); @@ -1739,7 +1736,7 @@ static UNUSED_FUNC bool doGetQueryPos(TSKEY key, SQInfo *pQInfo, SPointInterpoSu } static UNUSED_FUNC bool doSetDataInfo(SQInfo *pQInfo, SPointInterpoSupporter *pPointInterpSupporter, void *pMeterObj, - TSKEY nextKey) { + TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -2100,7 +2097,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI continue; } - int32_t colInBuf = 0;//pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; + int32_t colInBuf = 0; // pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); @@ -2198,8 +2195,8 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { } else if (isIntervalQuery(pQuery)) { // time window query, allocate one page for each table size_t s = pQInfo->groupInfo.numOfTables; num = MAX(s, INITIAL_RESULT_ROWS_VALUE); - } else { // for super table query, one page for each subset - num = 1;//pQInfo->pSidSet->numOfSubSet; + } else { // for super table query, one page for each subset + num = 1; // pQInfo->pSidSet->numOfSubSet; } assert(num > 0); @@ -2374,8 +2371,8 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl } if (r == BLK_DATA_NO_NEEDED) { - qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", - GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qTrace("QInfo:%p slot:%d, data block ignored, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); } else if (r == BLK_DATA_FILEDS_NEEDED) { if (tsdbRetrieveDataBlockStatisInfo(pRuntimeEnv->pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; @@ -2418,10 +2415,10 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - - TSKEY* keyList = (TSKEY *)pValue; + + TSKEY * keyList = (TSKEY *)pValue; int32_t firstPos = 0; - int32_t lastPos = num - 1; + int32_t lastPos = num - 1; if (order == TSDB_ORDER_DESC) { // find the first position which is smaller than the key @@ -2477,9 +2474,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", GET_QINFO_ADDR(pRuntimeEnv), pQuery->window.skey, pQuery->window.ekey, pQuery->lastKey, pQuery->order.order); - TsdbQueryHandleT pQueryHandle = pRuntimeEnv->scanFlag == MASTER_SCAN? pRuntimeEnv->pQueryHandle:pRuntimeEnv->pSecQueryHandle; + TsdbQueryHandleT pQueryHandle = + pRuntimeEnv->scanFlag == MASTER_SCAN ? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return 0; } @@ -2488,13 +2485,13 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { // todo extract methods if (isIntervalQuery(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == 0) { - TSKEY skey1, ekey1; - STimeWindow w = {0}; + TSKEY skey1, ekey1; + STimeWindow w = {0}; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { - getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, - &skey1, &ekey1, &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, + &ekey1, &w); pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } else { @@ -2506,39 +2503,41 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { pWindowResInfo->prevSKey = w.skey; } } - + // in case of prj/diff query, ensure the output buffer is sufficient to accomodate the results of current block if (!isIntervalQuery(pQuery) && !isGroupbyNormalCol(pQuery->pGroupbyExpr) && !isFixedOutputQuery(pQuery)) { - SResultRec* pRec = &pQuery->rec; - + SResultRec *pRec = &pQuery->rec; + if (pQuery->rec.capacity - pQuery->rec.rows < blockInfo.rows) { int32_t remain = pRec->capacity - pRec->rows; int32_t newSize = pRec->capacity + (blockInfo.rows - remain); - - for(int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + + for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t bytes = pQuery->pSelectExpr[i].resBytes; - - char* tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData)); - if (tmp == NULL) { // todo handle the oom + + char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData)); + if (tmp == NULL) { // todo handle the oom } else { - pQuery->sdata[i] = (SData*) tmp; + pQuery->sdata[i] = (SData *)tmp; } - + // set the pCtx output buffer position - pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows*bytes; + pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes; } - + pRec->capacity = newSize; } } SDataStatis *pStatis = NULL; - SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); + + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, - &pRuntimeEnv->windowResInfo, pDataBlock); + &pRuntimeEnv->windowResInfo, pDataBlock); - qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", - GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), + blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); // save last access position if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { @@ -2572,19 +2571,19 @@ static void updatelastkey(SQuery *pQuery, STableQueryInfo *pTableQInfo) { pTable * set tag value in SQLFunctionCtx * e.g.,tag information into input buffer */ -static void doSetTagValueInParam(void* tsdb, STableId id, int32_t tagColId, tVariant *param) { +static void doSetTagValueInParam(void *tsdb, STableId id, int32_t tagColId, tVariant *param) { tVariantDestroy(param); - char* val = NULL; + char * val = NULL; int16_t bytes = 0; - int16_t type = 0; + int16_t type = 0; tsdbGetTableTagVal(tsdb, id, tagColId, &type, &bytes, &val); tVariantCreateFromBinary(param, val, bytes, type); } -void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void* tsdb) { - SQuery * pQuery = pRuntimeEnv->pQuery; +void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void *tsdb) { + SQuery *pQuery = pRuntimeEnv->pQuery; SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; if (pQuery->numOfOutputCols == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { @@ -2599,8 +2598,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void* tsdb) { if (!TSDB_COL_IS_TAG(pCol->flag)) { continue; } - - + // todo use tag column index to optimize performance doSetTagValueInParam(tsdb, id, pCol->colId, &pRuntimeEnv->pCtx[idx].tag); } @@ -2609,8 +2607,8 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void* tsdb) { if (pFuncMsg->functionId == TSDB_FUNC_TS && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && pRuntimeEnv->pTSBuf != NULL) { assert(pFuncMsg->numOfParams == 1); - assert(0); // to do fix me -// doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); + assert(0); // to do fix me + // doSetTagValueInParam(pTagSchema, pFuncMsg->arg->argValue.i64, pMeterSidInfo, &pRuntimeEnv->pCtx[0].tag); } } } @@ -2816,9 +2814,9 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { int32_t ret = TSDB_CODE_SUCCESS; int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - + while (pQInfo->groupIndex < numOfGroups) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex); + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex); ret = mergeIntoGroupResultImpl(pQInfo, group); if (ret < 0) { // not enough disk space to save the data into disk return -1; @@ -2835,8 +2833,8 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { qTrace("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } - qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", - pQInfo, pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); + qTrace("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%lldms", pQInfo, + pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); return TSDB_CODE_SUCCESS; } @@ -2916,23 +2914,23 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pW return maxOutput; } -int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { +int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; size_t size = taosArrayGetSize(pGroup); - + tFilePage **buffer = (tFilePage **)pQuery->sdata; - int32_t *posList = calloc(size, sizeof(int32_t)); - + int32_t * posList = calloc(size, sizeof(int32_t)); + STableDataInfo **pTableList = malloc(POINTER_BYTES * size); // todo opt for the case of one table per group int32_t numOfTables = 0; for (int32_t i = 0; i < size; ++i) { - SPair* p = taosArrayGet(pGroup, i); - STableDataInfo* pInfo = p->sec; - + SPair * p = taosArrayGet(pGroup, i); + STableDataInfo *pInfo = p->sec; + SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, pInfo->pTableQInfo->tid); if (list.size > 0 && pInfo->pTableQInfo->windowResInfo.size > 0) { pTableList[numOfTables] = pInfo; @@ -2956,7 +2954,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { SResultInfo *pResultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); - + int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); @@ -3017,7 +3015,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray* pGroup) { if (buffer[0]->numOfElems != 0) { // there are data in buffer if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - + tfree(pTree); tfree(pTableList); tfree(posList); @@ -3128,7 +3126,7 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo * void disableFuncInReverseScan(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; int32_t order = pQuery->order.order; - + // group by normal columns and interval query on normal table SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { @@ -3136,7 +3134,7 @@ void disableFuncInReverseScan(SQueryRuntimeEnv *pRuntimeEnv) { } else { // for simple result of table query, for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; - + SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || @@ -3158,12 +3156,12 @@ void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { } if (isIntervalQuery(pQuery)) { -// for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { -// STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; -// SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; -// -// doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); -// } + // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + // + // doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); + // } } else { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); @@ -3175,7 +3173,8 @@ void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SWITCH_ORDER(pRuntimeEnv->pCtx[i].order);// = (pRuntimeEnv->pCtx[i].order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + SWITCH_ORDER(pRuntimeEnv->pCtx[i] + .order); // = (pRuntimeEnv->pCtx[i].order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; } } @@ -3209,7 +3208,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t) pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); + memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); } initCtxOutputBuf(pRuntimeEnv); @@ -3248,7 +3247,7 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; - + pRuntimeEnv->pCtx[j].currentStage = 0; aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); } @@ -3302,7 +3301,7 @@ void setQueryStatus(SQuery *pQuery, int8_t status) { bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - + bool toContinue = false; if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { // for each group result, call the finalize function for each column @@ -3345,74 +3344,72 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { return toContinue; } -static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv* pRuntimeEnv) { - SQuery* pQuery = pRuntimeEnv->pQuery; - +static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + SQueryStatusInfo info = { .status = pQuery->status, .windowIndex = pRuntimeEnv->windowResInfo.curIndex, .lastKey = pQuery->lastKey, .w = pQuery->window, + .curWindow = {.skey = pQuery->lastKey, .ekey = pQuery->window.ekey}, }; - + return info; } -static void setEnvBeforeReverseScan(SQueryRuntimeEnv* pRuntimeEnv, SQueryStatusInfo* pStatus) { - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - SQuery* pQuery = pRuntimeEnv->pQuery; - - // the step should be placed before order changed - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - +static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { + SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); + SQuery *pQuery = pRuntimeEnv->pQuery; + pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor if (pRuntimeEnv->pTSBuf) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); tsBufNextPos(pRuntimeEnv->pTSBuf); } - + // reverse order time range - pQuery->window.skey = pQuery->lastKey - step; - pQuery->window.ekey = pStatus->lastKey; // the start timestamp of current query - + pQuery->window = pStatus->curWindow; + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + SWITCH_ORDER(pQuery->order.order); SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); - + STsdbQueryCond cond = { - .twindow = pQuery->window, - .order = pQuery->order.order, - .colList = pQuery->colList, + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - + // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->groupInfo); - + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); disableFuncInReverseScan(pRuntimeEnv); } -static void clearEnvAfterReverseScan(SQueryRuntimeEnv* pRuntimeEnv, TSKEY lastKey, SQueryStatusInfo* pStatus) { - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { + SQuery *pQuery = pRuntimeEnv->pQuery; + SWITCH_ORDER(pQuery->order.order); switchCtxOrder(pRuntimeEnv); - + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pStatus->cur); if (pRuntimeEnv->pTSBuf) { pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; } - + SET_MASTER_SCAN_FLAG(pRuntimeEnv); - + // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query // during reverse scan - pQuery->lastKey = lastKey; + pQuery->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; pQuery->window = pStatus->w; } @@ -3422,7 +3419,7 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position - SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); + SQInfo * pQInfo = (SQInfo *)GET_QINFO_ADDR(pRuntimeEnv); SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv); SET_MASTER_SCAN_FLAG(pRuntimeEnv); @@ -3430,9 +3427,10 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { while (1) { doScanAllDataBlocks(pRuntimeEnv); - + if (pRuntimeEnv->scanFlag == MASTER_SCAN) { qstatus.status = pQuery->status; + qstatus.curWindow.ekey = pQuery->lastKey - step; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { @@ -3445,16 +3443,16 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } STsdbQueryCond cond = { - .twindow = {.skey = qstatus.lastKey, .ekey = pQuery->lastKey - step}, - .order = pQuery->order.order, - .colList = pQuery->colList, + .twindow = qstatus.curWindow, + .order = pQuery->order.order, + .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - + if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - + pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->groupInfo); pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; @@ -3466,19 +3464,18 @@ void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { return; } } - + if (!needReverseScan(pQuery)) { return; } - - TSKEY lastKey = pQuery->lastKey; + setEnvBeforeReverseScan(pRuntimeEnv, &qstatus); // reverse scan from current position - qTrace("QInfo:%p start to reverse scan", GET_QINFO_ADDR(pRuntimeEnv)); + qTrace("QInfo:%p start to reverse scan", pQInfo); doScanAllDataBlocks(pRuntimeEnv); - - clearEnvAfterReverseScan(pRuntimeEnv, lastKey, &qstatus); + + clearEnvAfterReverseScan(pRuntimeEnv, &qstatus); } void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { @@ -3588,7 +3585,7 @@ void restoreIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *p * @param pRuntimeEnv * @param pDataBlockInfo */ -void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STable* pTable, int32_t groupIdx, +void setExecutionContext(SQInfo *pQInfo, STableQueryInfo *pTableQueryInfo, STable *pTable, int32_t groupIdx, TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SWindowResInfo * pWindowResInfo = &pRuntimeEnv->windowResInfo; @@ -3642,7 +3639,7 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * } } -int32_t setAdditionalInfo(SQInfo *pQInfo, STable* pTable, STableQueryInfo *pTableQueryInfo) { +int32_t setAdditionalInfo(SQInfo *pQInfo, STable *pTable, STableQueryInfo *pTableQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; assert(pTableQueryInfo->lastKey > 0); @@ -3841,7 +3838,7 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) { int32_t numOfResult = doCopyToSData(pQInfo, result, orderType); pQuery->rec.rows += numOfResult; - + assert(pQuery->rec.rows <= pQuery->rec.capacity); } @@ -3860,11 +3857,12 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableDataInf } } -void stableApplyFunctionsOnBlock(SQueryRuntimeEnv* pRuntimeEnv, STableDataInfo *pTableDataInfo, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, SArray *pDataBlock, __block_search_fn_t searchFn) { - SQuery * pQuery = pRuntimeEnv->pQuery; - STableQueryInfo * pTableQueryInfo = pTableDataInfo->pTableQInfo; - SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; +void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo *pTableDataInfo, + SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SArray *pDataBlock, + __block_search_fn_t searchFn) { + SQuery * pQuery = pRuntimeEnv->pQuery; + STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { // numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, &forwardStep, pFields, pDataBlockInfo, pWindowResInfo); @@ -3901,8 +3899,8 @@ bool vnodeHasRemainResults(void *handle) { // query has completed if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - /*TSKEY ekey =*/ taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, - pQuery->slidingTimeUnit, pQuery->precision); + /*TSKEY ekey =*/taosGetRevisedEndKey(pQuery->window.ekey, pQuery->order.order, pQuery->intervalTime, + pQuery->slidingTimeUnit, pQuery->precision); // int32_t numOfTotal = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY // *)pRuntimeEnv->pInterpoBuf[0]->data, // remain, pQuery->intervalTime, ekey, @@ -3917,7 +3915,7 @@ bool vnodeHasRemainResults(void *handle) { } static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, tFilePage **pDataSrc, int32_t numOfRows, - int32_t outputRows) { + int32_t outputRows) { #if 0 SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = &pRuntimeEnv->pQuery; @@ -3956,11 +3954,11 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data SQuery *pQuery = pQInfo->runtimeEnv.pQuery; for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { int32_t bytes = pQuery->pSelectExpr[col].resBytes; - + memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); data += bytes * numOfRows; } - + // all data returned, set query over if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { setQueryStatus(pQuery, QUERY_OVER); @@ -4058,9 +4056,211 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { #endif } -int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; +static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed + pQuery->lastKey = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; + pQuery->limit.offset = 0; + return; + } + + if (QUERY_IS_ASC_QUERY(pQuery)) { + pQuery->pos = pQuery->limit.offset; + } else { + pQuery->pos = pBlockInfo->rows - pQuery->limit.offset - 1; + } + + assert(pQuery->pos >= 0 && pQuery->pos <= pBlockInfo->rows - 1); + + SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); + + // update the pQuery->limit.offset value, and pQuery->pos value + TSKEY *keys = (TSKEY *)pColInfoData->pData; + + // update the offset value + pQuery->lastKey = keys[pQuery->pos]; + pQuery->limit.offset = 0; + + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, + &pRuntimeEnv->windowResInfo, pDataBlock); + + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes); +} + +void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0) { + return; + } + + pQuery->pos = 0; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; + + while (tsdbNextDataBlock(pQueryHandle)) { + if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { + return; + } + + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + + if (pQuery->limit.offset > blockInfo.rows) { + pQuery->limit.offset -= blockInfo.rows; + pQuery->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; + pQuery->lastKey += step; + + qTrace("QInfo:%p skip rows:%d, offset:%" PRId64 "", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows, pQuery->limit.offset); + } else { // find the appropriated start position in current block + updateOffsetVal(pRuntimeEnv, &blockInfo); + break; + } + } +} + +static UNUSED_FUNC bool forwardQueryStartPosIfNeeded(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + + // if queried with value filter, do NOT forward query start position + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { + return true; + } + + if (pQuery->limit.offset > 0 && (!isTopBottomQuery(pQuery)) && pQuery->interpoType == TSDB_INTERPO_NONE) { + /* + * 1. for top/bottom query, the offset applies to the final result, not here + * 2. for interval without interpolation query we forward pQuery->intervalTime at a time for + * pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is + * not valid. otherwise, we only forward pQuery->limit.offset number of points + */ + if (isIntervalQuery(pQuery)) { + assert(pRuntimeEnv->windowResInfo.prevSKey == 0); + + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + + if (!tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { + // todo handle no data situation + } + + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle); + + if (QUERY_IS_ASC_QUERY(pQuery)) { + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, + &ekey1, &w); + pWindowResInfo->startTime = w.skey; + pWindowResInfo->prevSKey = w.skey; + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + TSKEY start = blockInfo.window.ekey - pQuery->intervalTime; + getAlignQueryTimeWindow(pQuery, start, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, &w); + + pWindowResInfo->startTime = pQuery->window.skey; + pWindowResInfo->prevSKey = w.skey; + } + + // the first time window + STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery); + + while (pQuery->limit.offset > 0) { + STimeWindow tw = win; + getNextTimeWindow(pQuery, &tw); + + // next time window starts from current data block + if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + // query completed + if ((tw.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (tw.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + setQueryStatus(pQuery, QUERY_COMPLETED); + break; + } + + tw = win; + SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); + + int32_t startPos = getNextQualifiedWindow(pRuntimeEnv, &tw, pWindowResInfo, &blockInfo, pColInfoData->pData, + binarySearchForKey); + assert(startPos >= 0); + pQuery->limit.offset -= 1; + + // set the abort info + pQuery->pos = startPos; + pQuery->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; + pWindowResInfo->prevSKey = tw.skey; + win = tw; + continue; + } else { + if (!tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { + setQueryStatus(pQuery, QUERY_COMPLETED); + break; + } + + blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle); + if ((blockInfo.window.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (blockInfo.window.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + setQueryStatus(pQuery, QUERY_COMPLETED); + break; + } + + // set the window that start from the next data block + TSKEY key = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.skey : blockInfo.window.ekey; + STimeWindow n = getActiveTimeWindow(pWindowResInfo, key, pQuery); + + // next data block are still covered by current time window + if (n.skey == win.skey && n.ekey == win.ekey) { + // do nothing + } else { + pQuery->limit.offset -= 1; + + // query completed + if ((n.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (n.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { + setQueryStatus(pQuery, QUERY_COMPLETED); + break; + } + + // set the abort info + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; + pQuery->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.skey : blockInfo.window.ekey; + pWindowResInfo->prevSKey = n.skey; + + win = n; + } + } + } + + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) || pQuery->limit.offset > 0) { + setQueryStatus(pQuery, QUERY_COMPLETED); + return false; + } else { + assert(0); + // if (IS_DISK_DATA_BLOCK(pQuery)) { + // getTimestampInDiskBlock(pRuntimeEnv, 0); + } + } + } else { // forward the start position for projection query + skipBlocks(&pQInfo->runtimeEnv); + if (pQuery->limit.offset > 0) { + setQueryStatus(pQuery, QUERY_COMPLETED); + return false; + } + } + + return true; +} + +int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t code = TSDB_CODE_SUCCESS; @@ -4071,52 +4271,52 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) pQuery->lastKey = pQuery->window.skey; STsdbQueryCond cond = { - .twindow = pQuery->window, - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, + .twindow = pQuery->window, + .order = pQuery->order.order, + .colList = pQuery->colList, + .numOfCols = pQuery->numOfCols, }; - + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->groupInfo); pQInfo->tsdb = tsdb; - + pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTSBuf = param; pRuntimeEnv->cur.vnodeIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; - + if (param != NULL) { int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); } // create runtime environment - code = setupQueryRuntimeEnv(pRuntimeEnv, NULL, pQuery->order.order); + code = setupQueryRuntimeEnv(pRuntimeEnv, pQuery->order.order); if (code != TSDB_CODE_SUCCESS) { return code; } pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, isSTableQuery); - + if (isSTableQuery) { int32_t rows = getInitialPageNum(pQInfo); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); if (code != TSDB_CODE_SUCCESS) { return code; } - + if (pQuery->intervalTime == 0) { int16_t type = TSDB_DATA_TYPE_NULL; - + if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_INT; // group id } - + initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 512, 4096, type); } - + } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { int32_t rows = getInitialPageNum(pQInfo); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize); @@ -4158,11 +4358,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void* tsdb, bool isSTableQuery) pointInterpSupporterSetData(pQInfo, &interpInfo); pointInterpSupporterDestroy(&interpInfo); - // todo move to other location - // if (!forwardQueryStartPosIfNeeded(pQInfo, pQInfo, dataInDisk, dataInCache)) { - // return TSDB_CODE_SUCCESS; - // } - int64_t rs = taosGetIntervalStartTimestamp(pQuery->window.skey, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision); taosInitInterpoInfo(&pRuntimeEnv->interpoInfo, pQuery->order.order, rs, 0, 0); @@ -4216,29 +4411,29 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - + int64_t st = taosGetTimestampMs(); - + TsdbQueryHandleT *pQueryHandle = pRuntimeEnv->pQueryHandle; while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(pQInfo)) { break; } - SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); - STableDataInfo* pTableDataInfo = NULL; - STable* pTable = NULL; - + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pQueryHandle); + STableDataInfo *pTableDataInfo = NULL; + STable * pTable = NULL; + // todo opt performance using hash table size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - for(int32_t i = 0; i < numOfGroup; ++i) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); - + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + size_t num = taosArrayGetSize(group); - for(int32_t j = 0; j < num; ++j) { - SPair* p = taosArrayGet(group, j); - STableDataInfo* pInfo = p->sec; - + for (int32_t j = 0; j < num; ++j) { + SPair * p = taosArrayGet(group, j); + STableDataInfo *pInfo = p->sec; + if (pInfo->pTableQInfo->tid == blockInfo.sid) { pTableDataInfo = p->sec; pTable = p->first; @@ -4246,7 +4441,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { } } } - + assert(pTableDataInfo != NULL && pTableDataInfo->pTableQInfo != NULL); STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; @@ -4261,7 +4456,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { } else { // interval query setIntervalQueryRange(pTableQueryInfo, pQInfo, nextKey); int32_t ret = setAdditionalInfo(pQInfo, pTable, pTableQueryInfo); - + if (ret != TSDB_CODE_SUCCESS) { pQInfo->code = ret; return taosGetTimestampMs() - st; @@ -4270,42 +4465,41 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { stableApplyFunctionsOnBlock(pRuntimeEnv, pTableDataInfo, &blockInfo, pStatis, pDataBlock, binarySearchForKey); } - + int64_t et = taosGetTimestampMs(); return et - st; } static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); - SPair* p = taosArrayGet(group, index); - - STable* pTable = p->first; - STableDataInfo* pInfo = p->sec; - + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); + SPair * p = taosArrayGet(group, index); + + STable * pTable = p->first; + STableDataInfo *pInfo = p->sec; + setTagVal(pRuntimeEnv, pTable->tableId, pQInfo->tsdb); - + qTrace("QInfo:%p query on (%d): uid:%" PRIu64 ", tid:%d, qrange:%" PRId64 "-%" PRId64, pQInfo, index, pTable->tableId.uid, pInfo->pTableQInfo->lastKey, pInfo->pTableQInfo->win.ekey); - + STsdbQueryCond cond = { .twindow = {pInfo->pTableQInfo->lastKey, pInfo->pTableQInfo->win.ekey}, - .order = pQuery->order.order, + .order = pQuery->order.order, .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - - - SArray* g1 = taosArrayInit(1, POINTER_BYTES); - SArray* tx = taosArrayInit(1, sizeof(SPair)); - + + SArray *g1 = taosArrayInit(1, POINTER_BYTES); + SArray *tx = taosArrayInit(1, sizeof(SPair)); + taosArrayPush(tx, p); taosArrayPush(g1, &tx); STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1}; - + // include only current table pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp); @@ -4313,7 +4507,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->cur.vnodeIndex == -1) { int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); - + // failed to find data with the specified tag value if (elem.vnode < 0) { return false; @@ -4377,11 +4571,11 @@ static UNUSED_FUNC int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, i */ static void sequentialTableProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; setQueryStatus(pQuery, QUERY_COMPLETED); - + size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - + if (isPointInterpoQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); @@ -4441,10 +4635,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } } #endif - + } else { createTableDataInfo(pQInfo); - + /* * 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query * if the subgroup index is larger than 0, results generated by group by tbname,k is existed. @@ -4453,67 +4647,68 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (pQInfo->groupIndex > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); pQuery->rec.total += pQuery->rec.rows; - + if (pQuery->rec.rows > 0) { return; } } - + // all data have returned already if (pQInfo->tableIndex >= pQInfo->groupInfo.numOfTables) { return; } - + resetCtxOutputBuf(pRuntimeEnv); resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); - - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); - assert(taosArrayGetSize(group) == pQInfo->groupInfo.numOfTables && 1 == taosArrayGetSize(pQInfo->groupInfo.pGroupList)); - + + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, 0); + assert(taosArrayGetSize(group) == pQInfo->groupInfo.numOfTables && + 1 == taosArrayGetSize(pQInfo->groupInfo.pGroupList)); + while (pQInfo->tableIndex < pQInfo->groupInfo.numOfTables) { if (isQueryKilled(pQInfo)) { return; } - - SPair *p = taosArrayGet(group, pQInfo->tableIndex); - STableDataInfo* pInfo = p->sec; - + + SPair * p = taosArrayGet(group, pQInfo->tableIndex); + STableDataInfo *pInfo = p->sec; + TSKEY skey = pInfo->pTableQInfo->lastKey; if (skey > 0) { pQuery->window.skey = skey; } - + if (!multiTableMultioutputHelper(pQInfo, pQInfo->tableIndex)) { pQInfo->tableIndex++; continue; } - -// SPointInterpoSupporter pointInterpSupporter = {0}; - + + // SPointInterpoSupporter pointInterpSupporter = {0}; + // TODO handle the limit problem if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { -// forwardQueryStartPosition(pRuntimeEnv); - + // skipBlocks(pRuntimeEnv); + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { pQInfo->tableIndex++; continue; } } - + scanAllDataBlocks(pRuntimeEnv); - + pQuery->rec.rows = getNumOfResult(pRuntimeEnv); doSkipResults(pRuntimeEnv); - + // the limitation of output result is reached, set the query completed if (doRevisedResultsByLimit(pQInfo)) { pQInfo->tableIndex = pQInfo->groupInfo.numOfTables; break; } - + // enable execution for next table, when handling the projection query enableExecutionForNextTable(pRuntimeEnv); - + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { /* * query range is identical in terms of all meters involved in query, @@ -4523,30 +4718,30 @@ static void sequentialTableProcess(SQInfo *pQInfo) { */ pQInfo->tableIndex++; pInfo->pTableQInfo->lastKey = pQuery->lastKey; - + // if the buffer is full or group by each table, we need to jump out of the loop if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL) /*|| isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)*/) { break; } - + } else { // forward query range pQuery->window.skey = pQuery->lastKey; - + // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter if (pQuery->rec.rows == 0) { assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); continue; } else { -// pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; -// // buffer is full, wait for the next round to retrieve data from current meter -// assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); -// break; + // pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; + // // buffer is full, wait for the next round to retrieve data from current meter + // assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); + // break; } } } } - + /* * 1. super table projection query, group-by on normal columns query, ts-comp query * 2. point interpolation query, last row query @@ -4561,123 +4756,126 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (isTSCompQuery(pQuery)) { finalizeQueryResult(pRuntimeEnv); } - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; } - + // todo refactor if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; pStatus->closed = true; // enable return all results for group by normal columns - + SWindowResult *pResult = &pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); } } - + pQInfo->groupIndex = 0; pQuery->rec.rows = 0; copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult); } - + pQuery->rec.total += pQuery->rec.rows; - - qTrace( "QInfo %p, numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%d totalReturn:%d," - " offset:%" PRId64, pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, - pQuery->rec.rows, pQuery->rec.total, pQuery->limit.offset); + + qTrace( + "QInfo %p, numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%d totalReturn:%d," + " offset:%" PRId64, + pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total, + pQuery->limit.offset); } -static void createTableDataInfo(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - +static void createTableDataInfo(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + // todo make sure the table are added the reference count to gauranteed that all involved tables are valid size_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - + int32_t index = 0; for (int32_t i = 0; i < numOfGroups; ++i) { // load all meter meta info - SArray *group = *(SArray**) taosArrayGet(pQInfo->groupInfo.pGroupList, i); - + SArray *group = *(SArray **)taosArrayGet(pQInfo->groupInfo.pGroupList, i); + size_t s = taosArrayGetSize(group); - for(int32_t j = 0; j < s; ++j) { - SPair* p = (SPair*) taosArrayGet(group, j); - + for (int32_t j = 0; j < s; ++j) { + SPair *p = (SPair *)taosArrayGet(group, j); + // STableDataInfo has been created for each table if (p->sec != NULL) { // todo refactor return; } - - STableDataInfo* pInfo = calloc(1, sizeof(STableDataInfo)); - + + STableDataInfo *pInfo = calloc(1, sizeof(STableDataInfo)); + setTableDataInfo(pInfo, index, i); - pInfo->pTableQInfo = createTableQueryInfo(&pQInfo->runtimeEnv, ((STable*)(p->first))->tableId.tid, pQuery->window); - + pInfo->pTableQInfo = + createTableQueryInfo(&pQInfo->runtimeEnv, ((STable *)(p->first))->tableId.tid, pQuery->window); + p->sec = pInfo; - + index += 1; } } } static void prepareQueryInfoForReverseScan(SQInfo *pQInfo) { -// SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - -// for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { -// STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; -// changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); -// } + // SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // changeMeterQueryInfoForSuppleQuery(pQuery, pTableQueryInfo); + // } } -static void doSaveContext(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void doSaveContext(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); disableFuncForReverseScan(pQInfo, pQuery->order.order); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1u; } - + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); prepareQueryInfoForReverseScan(pQInfo); } -static void doRestoreContext(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void doRestoreContext(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; } - + switchCtxOrder(pRuntimeEnv); SET_MASTER_SCAN_FLAG(pRuntimeEnv); } -static void doCloseAllTimeWindowAfterScan(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - +static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (isIntervalQuery(pQuery)) { -// for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { -// STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; -// closeAllTimeWindow(&pTableQueryInfo->windowResInfo); -// } + // for (int32_t i = 0; i < pQInfo->groupInfo.numOfTables; ++i) { + // STableQueryInfo *pTableQueryInfo = pQInfo->pTableDataInfo[i].pTableQInfo; + // closeAllTimeWindow(&pTableQueryInfo->windowResInfo); + // } size_t numOfGroup = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - for(int32_t i = 0; i < numOfGroup; ++i) { - SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); - + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray *group = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + size_t num = taosArrayGetSize(group); - for(int32_t j = 0; j < num; ++j) { - SPair* p = taosArrayGet(group, j); - STableDataInfo* pInfo = p->sec; - + for (int32_t j = 0; j < num; ++j) { + SPair * p = taosArrayGet(group, j); + STableDataInfo *pInfo = p->sec; + closeAllTimeWindow(&pInfo->pTableQInfo->windowResInfo); } } @@ -4714,48 +4912,48 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { qTrace("QInfo:%p current:%lld, total:%lld", pQInfo, pQuery->rec.rows, pQuery->rec.total); return; } - - qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, pQuery->window.skey, - pQuery->window.ekey, pQuery->order.order); - + + qTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, + pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); + // create the query support structures createTableDataInfo(pQInfo); - + // do check all qualified data blocks int64_t el = queryOnDataBlocks(pQInfo); qTrace("QInfo:%p forward scan completed, elapsed time: %lldms, reversed scan start, order:%d", pQInfo, el, pQuery->order.order ^ 1u); - + // query error occurred or query is killed, abort current execution if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - + // close all time window results doCloseAllTimeWindowAfterScan(pQInfo); - + if (needReverseScan(pQuery)) { doSaveContext(pQInfo); - + el = queryOnDataBlocks(pQInfo); qTrace("QInfo:%p reversed scan completed, elapsed time: %lldms", pQInfo, el); - + doRestoreContext(pQInfo); } else { qTrace("QInfo:%p no need to do reversed scan, query completed", pQInfo); } - + setQueryStatus(pQuery, QUERY_COMPLETED); - + if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { qTrace("QInfo:%p query killed or error occurred, code:%d, abort", pQInfo, pQInfo->code); return; } - + if (isIntervalQuery(pQuery) || isSumAvgRateQuery(pQuery)) { -// assert(pSupporter->groupIndex == 0 && pSupporter->numOfGroupResultPages == 0); - + // assert(pSupporter->groupIndex == 0 && pSupporter->numOfGroupResultPages == 0); + if (mergeIntoGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { copyResToQueryResultBuf(pQInfo, pQuery); @@ -4766,7 +4964,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { } else { // not a interval query copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); } - + // handle the limitation of output buffer qTrace("QInfo:%p points returned:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } @@ -4802,12 +5000,19 @@ static void tableFixedOutputProcess(SQInfo *pQInfo) { static void tableMultiOutputProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery * pQuery = pRuntimeEnv->pQuery; // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); } + + // skip blocks without load the actual data block from file if no filter condition present + skipBlocks(&pQInfo->runtimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { + setQueryStatus(pQuery, QUERY_COMPLETED); + return; + } while (1) { scanAllDataBlocks(pRuntimeEnv); @@ -4838,13 +5043,10 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { doRevisedResultsByLimit(pQInfo); if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, - pQInfo, pQuery->lastKey, pQuery->window.ekey); + qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->lastKey, + pQuery->window.ekey); } -// qTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, -// pMeterObj->sid, pMeterObj->meterId, pQuery->size, pQInfo->size, pQInfo->pointsReturned); - if (!isTSCompQuery(pQuery)) { assert(pQuery->rec.rows <= pQuery->rec.capacity); } @@ -4875,7 +5077,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= c; } - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED|QUERY_RESBUF_FULL)) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_RESBUF_FULL)) { break; } } @@ -4912,8 +5114,8 @@ static void tableIntervalProcess(SQInfo *pQInfo) { } numOfInterpo = 0; - pQuery->rec.rows = vnodeQueryResultInterpolate( - pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, pQuery->rec.rows, &numOfInterpo); + pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pInterpoBuf, + pQuery->rec.rows, &numOfInterpo); qTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.rows); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { @@ -4937,10 +5139,10 @@ static void tableIntervalProcess(SQInfo *pQInfo) { pQInfo->pointsInterpo += numOfInterpo; } -static void tableQueryImpl(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - +static void tableQueryImpl(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery * pQuery = pRuntimeEnv->pQuery; + if (vnodeHasRemainResults(pQInfo)) { /* * There are remain results that are not returned due to result interpolation @@ -4950,31 +5152,30 @@ static void tableQueryImpl(SQInfo* pQInfo) { int32_t remain = taosNumOfRemainPoints(&pRuntimeEnv->interpoInfo); pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); - + doRevisedResultsByLimit(pQInfo); - + pQInfo->pointsInterpo += numOfInterpo; qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); return; } - + // here we have scan all qualified data in both data file and cache if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { // continue to get push data from the group result if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || - ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { - + ((isIntervalQuery(pQuery) && pQuery->rec.total < pQuery->limit.limit))) { // todo limit the output for interval query? pQuery->rec.rows = 0; pQInfo->groupIndex = 0; // always start from 0 - + if (pRuntimeEnv->windowResInfo.size > 0) { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); pQuery->rec.rows += pQuery->rec.rows; - + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); - + if (pQuery->rec.rows > 0) { qTrace("QInfo:%p %d rows returned from group results, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); sem_post(&pQInfo->dataReady); @@ -4982,17 +5183,17 @@ static void tableQueryImpl(SQInfo* pQInfo) { } } } - + qTrace("QInfo:%p query over, %d rows are returned", pQInfo, pQuery->rec.total); // vnodePrintQueryStatistics(pSupporter); sem_post(&pQInfo->dataReady); return; } - + // number of points returned during this query pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (isIntervalQuery(pQuery) || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) tableIntervalProcess(pQInfo); @@ -5002,48 +5203,50 @@ static void tableQueryImpl(SQInfo* pQInfo) { assert(pQuery->checkBuffer == 1); tableMultiOutputProcess(pQInfo); } - + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); assert(pQInfo->groupInfo.numOfTables == 1); - + /* check if query is killed or not */ if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p query is killed", pQInfo); } else { -// STableId* pTableId = taosArrayGet(pQInfo->groupInfo, 0); -// qTrace("QInfo:%p uid:%" PRIu64 " tid:%d, query completed, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", -// pQInfo, pTableId->uid, pTableId->tid, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); + // STableId* pTableId = taosArrayGet(pQInfo->groupInfo, 0); + // qTrace("QInfo:%p uid:%" PRIu64 " tid:%d, query completed, %" PRId64 " rows returned, numOfTotal:%" PRId64 " + // rows", + // pQInfo, pTableId->uid, pTableId->tid, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - + sem_post(&pQInfo->dataReady); } -static void stableQueryImpl(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; +static void stableQueryImpl(SQInfo *pQInfo) { + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; pQuery->rec.rows = 0; - + int64_t st = taosGetTimestampUs(); - + if (isIntervalQuery(pQuery) || (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || - isGroupbyNormalCol(pQuery->pGroupbyExpr)); - + isGroupbyNormalCol(pQuery->pGroupbyExpr)); + sequentialTableProcess(pQInfo); } - + // record the total elapsed time pQInfo->elapsedTime += (taosGetTimestampUs() - st); -// taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); - + // taosInterpoSetStartInfo(&pQInfo->runtimeEnv.interpoInfo, pQuery->size, pQInfo->query.interpoType); + if (pQuery->rec.rows == 0) { - qTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, pQInfo->groupInfo.numOfTables, pQuery->rec.total); -// vnodePrintQueryStatistics(pSupporter); + qTrace("QInfo:%p over, %d tables queried, %d points are returned", pQInfo, pQInfo->groupInfo.numOfTables, + pQuery->rec.total); + // vnodePrintQueryStatistics(pSupporter); } - + sem_post(&pQInfo->dataReady); } @@ -5095,32 +5298,32 @@ static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { return 0; } -static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** pTableIdList) { +static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { assert(pQueryMsg->numOfTables > 0); - + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableId)); - + STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - + STableId id = {.uid = pTableIdInfo->uid, .tid = pTableIdInfo->sid}; taosArrayPush(*pTableIdList, &id); - + pMsg += sizeof(STableIdInfo); - + for (int32_t j = 1; j < pQueryMsg->numOfTables; ++j) { pTableIdInfo = (STableIdInfo *)pMsg; - + pTableIdInfo->sid = htonl(pTableIdInfo->sid); pTableIdInfo->uid = htobe64(pTableIdInfo->uid); pTableIdInfo->key = htobe64(pTableIdInfo->key); - + taosArrayPush(*pTableIdList, pTableIdInfo); pMsg += sizeof(STableIdInfo); } - + return pMsg; } @@ -5133,28 +5336,28 @@ static char* createTableIdList(SQueryTableMsg* pQueryMsg, char* pMsg, SArray** p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, - char** tagCond, SColIndex** groupbyCols) { - pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); + char **tagCond, SColIndex **groupbyCols) { + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); - pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); - pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); - pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); - pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); - pQueryMsg->limit = htobe64(pQueryMsg->limit); - pQueryMsg->offset = htobe64(pQueryMsg->offset); - - pQueryMsg->order = htons(pQueryMsg->order); - pQueryMsg->orderColId = htons(pQueryMsg->orderColId); - pQueryMsg->queryType = htons(pQueryMsg->queryType); + pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); + pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime); + pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); + pQueryMsg->limit = htobe64(pQueryMsg->limit); + pQueryMsg->offset = htobe64(pQueryMsg->offset); - pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); + pQueryMsg->order = htons(pQueryMsg->order); + pQueryMsg->orderColId = htons(pQueryMsg->orderColId); + pQueryMsg->queryType = htons(pQueryMsg->queryType); + + pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); - pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); - pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); - pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); - pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); + pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); // query msg safety check if (validateQueryMsg(pQueryMsg) != 0) { @@ -5164,10 +5367,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, char *pMsg = (char *)(pQueryMsg->colList) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { - SColumnInfo* pColInfo = &pQueryMsg->colList[col]; - + SColumnInfo *pColInfo = &pQueryMsg->colList[col]; + pColInfo->colId = htons(pColInfo->colId); - pColInfo->type = htons(pColInfo->type); + pColInfo->type = htons(pColInfo->type); pColInfo->bytes = htons(pColInfo->bytes); pColInfo->numOfFilters = htons(pColInfo->numOfFilters); @@ -5211,15 +5414,15 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, (*pExpr)[i] = pExprMsg; pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); - pExprMsg->functionId = htons(pExprMsg->functionId); - pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); pMsg += sizeof(SSqlFuncExprMsg); for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { - pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); + pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); pExprMsg->arg[j].argBytes = htons(pExprMsg->arg[j].argBytes); if (pExprMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) { @@ -5252,26 +5455,26 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->colNameList = (int64_t)pMsg; pMsg += pQueryMsg->colNameLen; } - + pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns - *groupbyCols = malloc(pQueryMsg->numOfGroupCols*sizeof(SColIndex)); - - for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - (*groupbyCols)[i].colId = *(int16_t*) pMsg; + *groupbyCols = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); + + for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { + (*groupbyCols)[i].colId = *(int16_t *)pMsg; pMsg += sizeof((*groupbyCols)[i].colId); - - (*groupbyCols)[i].colIndex = *(int16_t*) pMsg; + + (*groupbyCols)[i].colIndex = *(int16_t *)pMsg; pMsg += sizeof((*groupbyCols)[i].colIndex); - (*groupbyCols)[i].flag = *(int16_t*) pMsg; + (*groupbyCols)[i].flag = *(int16_t *)pMsg; pMsg += sizeof((*groupbyCols)[i].flag); memcpy((*groupbyCols)[i].name, pMsg, tListLen(groupbyCols[i]->name)); pMsg += tListLen((*groupbyCols)[i].name); } - + pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); pQueryMsg->orderType = htons(pQueryMsg->orderType); } @@ -5284,22 +5487,23 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { v[i] = htobe64(v[i]); } - + pMsg += sizeof(int64_t) * pQueryMsg->numOfOutputCols; } - + // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { *tagCond = calloc(1, pQueryMsg->tagCondLen); memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); } - - qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, ts order:%d, " - "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 ", offset:%" PRId64, - pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, - pQueryMsg->numOfGroupCols, pQueryMsg->order, pQueryMsg->numOfOutputCols, - pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType, pQueryMsg->tsLen, - pQueryMsg->limit, pQueryMsg->offset); + + qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 + ", numOfGroupbyTagCols:%d, ts order:%d, " + "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 + ", offset:%" PRId64, + pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, + pQueryMsg->order, pQueryMsg->numOfOutputCols, pQueryMsg->numOfCols, pQueryMsg->intervalTime, + pQueryMsg->interpoType, pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset); return 0; } @@ -5355,7 +5559,8 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs return TSDB_CODE_SUCCESS; } -static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr, SSqlFuncExprMsg** pExprMsg) { +static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr, + SSqlFuncExprMsg **pExprMsg) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; @@ -5434,7 +5639,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct return TSDB_CODE_SUCCESS; } -static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex* pColIndex, int32_t *code) { +static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { if (pQueryMsg->numOfGroupCols == 0) { return NULL; } @@ -5538,31 +5743,31 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_SUCCESS; } -static void doUpdateExprColumnIndex(SQuery* pQuery) { +static void doUpdateExprColumnIndex(SQuery *pQuery) { assert(pQuery->pSelectExpr != NULL && pQuery != NULL); -// int32_t i = 0, j = 0; -// while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) { -// if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIndex = (int16_t)j++; -// } else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) { -// pQuery->colList[i++].colIndex = -1; -// } else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) { -// j++; -// } -// } + // int32_t i = 0, j = 0; + // while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) { + // if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) { + // pQuery->colList[i++].colIndex = (int16_t)j++; + // } else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) { + // pQuery->colList[i++].colIndex = -1; + // } else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) { + // j++; + // } + // } -// while (i < pQuery->numOfCols) { -// pQuery->colList[i++].colIndex = -1; // not such column in current meter -// } - - for(int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - SSqlFuncExprMsg* pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; + // while (i < pQuery->numOfCols) { + // pQuery->colList[i++].colIndex = -1; // not such column in current meter + // } + + for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + SSqlFuncExprMsg *pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) { continue; } - - SColIndex* pColIndexEx = &pSqlExprMsg->colInfo; - for(int32_t f = 0; f < pQuery->numOfCols; ++f) { + + SColIndex *pColIndexEx = &pSqlExprMsg->colInfo; + for (int32_t f = 0; f < pQuery->numOfCols; ++f) { if (pColIndexEx->colId == pQuery->colList[f].info.colId) { pColIndexEx->colIndex = f; break; @@ -5610,7 +5815,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i].info = pQueryMsg->colList[i]; - + SColumnInfo *pColInfo = &pQuery->colList[i].info; pColInfo->filters = NULL; // if (colList[i].numOfFilters > 0) { @@ -5629,7 +5834,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou assert(pExprs[col].resBytes > 0); pQuery->rowSize += pExprs[col].resBytes; } - + doUpdateExprColumnIndex(pQuery); int32_t ret = vnodeCreateFilterInfo(pQInfo, pQuery); @@ -5646,7 +5851,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou // set the output buffer capacity pQuery->rec.capacity = 4096; pQuery->rec.threshold = 4000; - + for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { assert(pExprs[col].interResBytes >= pExprs[col].resBytes); @@ -5673,18 +5878,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->groupInfo = *groupInfo; pQuery->pos = -1; - + pQuery->window.skey = pQueryMsg->window.skey; pQuery->window.ekey = pQueryMsg->window.ekey; - pQuery->lastKey = pQuery->window.skey; - + pQuery->lastKey = pQuery->window.skey; + if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); goto _clean_memory; } - + vnodeParametersSafetyCheck(pQuery); - + qTrace("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo); return pQInfo; @@ -5719,30 +5924,30 @@ static bool isValidQInfo(void *param) { * pQInfo->signature may be changed by another thread, so we assign value of signature * into local variable, then compare by using local variable */ - uint64_t sig = (uint64_t) pQInfo->signature; + uint64_t sig = (uint64_t)pQInfo->signature; return (sig == (uint64_t)pQInfo); } -static void freeQInfo(SQInfo *pQInfo); -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void* tsdb, SQInfo *pQInfo, bool isSTable) { +static void freeQInfo(SQInfo *pQInfo); +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, SQInfo *pQInfo, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - + STSBuf *pTSBuf = NULL; if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *)pQueryMsg + pQueryMsg->tsOffset; pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); - + tsBufResetPos(pTSBuf); tsBufNextPos(pTSBuf); } - + // only the successful complete requries the sem_post/over = 1 operations. if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { qTrace("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); - + sem_post(&pQInfo->dataReady); setQueryStatus(pQuery, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; @@ -5767,67 +5972,67 @@ static void freeQInfo(SQInfo *pQInfo) { if (!isValidQInfo(pQInfo)) { return; } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; setQueryKilled(pQInfo); - + qTrace("QInfo:%p start to free QInfo", pQInfo); for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { tfree(pQuery->sdata[col]); } - + sem_destroy(&(pQInfo->dataReady)); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; if (pColFilter->numOfFilters > 0) { tfree(pColFilter->pFilters); } } - + tfree(pQuery->pFilterInfo); tfree(pQuery->colList); tfree(pQuery->sdata); - + if (pQuery->pSelectExpr != NULL) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { SSqlBinaryExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; - + if (pBinExprInfo->numOfCols > 0) { tfree(pBinExprInfo->pReqColumns); tExprTreeDestroy(&pBinExprInfo->pBinExpr, NULL); } } - + tfree(pQuery->pSelectExpr); } - + if (pQuery->defaultVal != NULL) { tfree(pQuery->defaultVal); } - + tfree(pQuery->pGroupbyExpr); tfree(pQuery); - + int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* p = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + for (int32_t i = 0; i < numOfGroups; ++i) { + SArray *p = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); taosArrayDestroy(p); } - + taosArrayDestroy(pQInfo->groupInfo.pGroupList); - + qTrace("QInfo:%p QInfo is freed", pQInfo); - + // destroy signature, in order to avoid the query process pass the object safety check memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + /* * get the file size and set the numOfRows to be the file size, since for tsComp query, * the returned row size is equalled to 1 @@ -5850,20 +6055,20 @@ static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // the remained number of retrieved rows, not the interpolated result SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - + // load data from file to msg buffer if (isTSCompQuery(pQuery)) { int32_t fd = open(pQuery->sdata[0]->data, O_RDONLY, 0666); - + // make sure file exist if (FD_VALID(fd)) { size_t s = lseek(fd, 0, SEEK_END); qTrace("QInfo:%p ts comp data return, file:%s, size:%zu", pQInfo, pQuery->sdata[0]->data, s); - + lseek(fd, 0, SEEK_SET); read(fd, data, s); close(fd); - + unlink(pQuery->sdata[0]->data); } else { qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, @@ -5872,25 +6077,25 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { } else { doCopyQueryResultToMsg(pQInfo, pQuery->rec.rows, data); } - + pQuery->rec.total += pQuery->rec.rows; qTrace("QInfo:%p current:%d, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); - + return TSDB_CODE_SUCCESS; - + // todo if interpolation exists, the result may be dump to client by several rounds } -int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { +int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) { assert(pQueryMsg != NULL); int32_t code = TSDB_CODE_SUCCESS; - - char* tagCond = NULL; - SArray *pTableIdList = NULL; - SSqlFuncExprMsg** pExprMsg = NULL; - SColIndex* pGroupColIndex = NULL; - + + char * tagCond = NULL; + SArray * pTableIdList = NULL; + SSqlFuncExprMsg **pExprMsg = NULL; + SColIndex * pGroupColIndex = NULL; + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &pGroupColIndex)) != TSDB_CODE_SUCCESS) { return code; } @@ -5917,37 +6122,38 @@ int32_t qCreateQueryInfo(void* tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { goto _query_over; } - - bool isSTableQuery = false; + + bool isSTableQuery = false; STableGroupInfo groupInfo = {0}; - + if ((pQueryMsg->queryType & TSDB_QUERY_TYPE_STABLE_QUERY) != 0) { isSTableQuery = true; - - STableId* id = taosArrayGet(pTableIdList, 0); - id->uid = -1; //todo fix me - - /*int32_t ret =*/ tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, &groupInfo, pGroupColIndex, pQueryMsg->numOfGroupCols); - if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query + + STableId *id = taosArrayGet(pTableIdList, 0); + id->uid = -1; // todo fix me + + /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, &groupInfo, pGroupColIndex, + pQueryMsg->numOfGroupCols); + if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query code = TSDB_CODE_SUCCESS; goto _query_over; } } else { assert(taosArrayGetSize(pTableIdList) == 1); - - STableId* id = taosArrayGet(pTableIdList, 0); + + STableId *id = taosArrayGet(pTableIdList, 0); if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) { goto _query_over; } } - + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo); if ((*pQInfo) == NULL) { code = TSDB_CODE_SERV_OUT_OF_MEMORY; } - + code = initQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); - + _query_over: taosArrayDestroy(pTableIdList); @@ -5962,36 +6168,36 @@ void qDestroyQueryInfo(qinfo_t pQInfo) { } void qTableQuery(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || pQInfo->signature != pQInfo) { qTrace("%p freed abort query", pQInfo); return; } - + if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p it is already killed, abort", pQInfo); return; } - + qTrace("QInfo:%p query task is launched", pQInfo); - + if (pQInfo->runtimeEnv.stableQuery) { stableQueryImpl(pQInfo); } else { tableQueryImpl(pQInfo); } - + // vnodeDecRefCount(pQInfo); } int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (isQueryKilled(pQInfo)) { qTrace("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); @@ -6000,19 +6206,19 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { sem_wait(&pQInfo->dataReady); qTrace("QInfo:%p retrieve result info, rowsize:%d, rows:%d, code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, - pQInfo->code); - + pQInfo->code); + return pQInfo->code; } bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || pQInfo->signature != pQInfo || pQInfo->code != TSDB_CODE_SUCCESS) { return false; } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { return false; } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { @@ -6024,21 +6230,21 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { } } -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen) { - SQInfo* pQInfo = (SQInfo*) qinfo; - +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen) { + SQInfo *pQInfo = (SQInfo *)qinfo; + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_INVALID_QHANDLE; } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - size_t size = getResultSize(pQInfo, &pQuery->rec.rows); + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + size_t size = getResultSize(pQInfo, &pQuery->rec.rows); *contLen = size + sizeof(SRetrieveTableRsp); - + // todo handle failed to allocate memory *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); (*pRsp)->numOfRows = htonl(pQuery->rec.rows); - + int32_t code = pQInfo->code; if (code == TSDB_CODE_SUCCESS) { (*pRsp)->offset = htobe64(pQuery->limit.offset); @@ -6047,23 +6253,23 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* co (*pRsp)->offset = 0; (*pRsp)->useconds = 0; } - + if (pQuery->rec.rows > 0 && code == TSDB_CODE_SUCCESS) { code = doDumpQueryResult(pQInfo, (*pRsp)->data); } else { setQueryStatus(pQuery, QUERY_OVER); code = pQInfo->code; } - + if (isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { - (*pRsp)->completed = 1; // notify no more result to client + (*pRsp)->completed = 1; // notify no more result to client } - + return code; - -// if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { -// qTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); -// vnodeDecRefCount(pObj->qhandle); -// pObj->qhandle = NULL; -// } + + // if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { + // qTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); + // vnodeDecRefCount(pObj->qhandle); + // pObj->qhandle = NULL; + // } } From 773432c1191717f983b02b3378c4d5fc66dbb3d3 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 21 Apr 2020 14:01:52 +0800 Subject: [PATCH 02/63] [td-169] fix bugs in interval query with limit/offset conditions --- src/query/src/queryExecutor.c | 234 +++++++++++++++------------------- 1 file changed, 105 insertions(+), 129 deletions(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index e958a53e87..c1208dbc0b 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -765,8 +765,8 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus } static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNextWin, - SWindowResInfo *pWindowResInfo, SDataBlockInfo *pDataBlockInfo, - TSKEY *primaryKeys, __block_search_fn_t searchFn) { + SDataBlockInfo *pDataBlockInfo, TSKEY *primaryKeys, + __block_search_fn_t searchFn) { SQuery *pQuery = pRuntimeEnv->pQuery; while (1) { @@ -945,8 +945,7 @@ static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStati STimeWindow nextWin = win; while (1) { - int32_t startPos = - getNextQualifiedWindow(pRuntimeEnv, &nextWin, pWindowResInfo, pDataBlockInfo, primaryKeyCol, searchFn); + int32_t startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, primaryKeyCol, searchFn); if (startPos < 0) { break; } @@ -1357,10 +1356,10 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY } else { pCtx->preAggVals.isSet = false; } - - pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery)? pQuery->pos : 0; - pCtx->size = QUERY_IS_ASC_QUERY(pQuery)? size - pQuery->pos : pQuery->pos + 1; - + + pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos : 0; + pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? size - pQuery->pos : pQuery->pos + 1; + uint32_t status = aAggs[functionId].nStatus; if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { pCtx->ptsList = &tsCol[pCtx->startOffset]; @@ -1370,7 +1369,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY // last_dist or first_dist function // store the first&last timestamp into the intermediate buffer [1], the true // value may be null but timestamp will never be null -// pCtx->ptsList = tsCol; + // pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { /* @@ -1386,12 +1385,12 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY pTWAInfo->EKey = pQuery->window.ekey; } -// pCtx->ptsList = tsCol; + // pCtx->ptsList = tsCol; } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[1].pz = param; } - + #if defined(_DEBUG_VIEW) // int64_t *tsList = (int64_t *)primaryColumnData; // int64_t s = tsList[0]; @@ -2496,8 +2495,8 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { pWindowResInfo->prevSKey = w.skey; } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - TSKEY start = blockInfo.window.ekey - pQuery->intervalTime; - getAlignQueryTimeWindow(pQuery, start, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, &w); + getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &skey1, + &ekey1, &w); pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; @@ -2531,7 +2530,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SDataStatis *pStatis = NULL; SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); - + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, &pRuntimeEnv->windowResInfo, pDataBlock); @@ -4059,13 +4058,13 @@ void vnodePrintQueryStatistics(SQInfo *pQInfo) { static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - + if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed - pQuery->lastKey = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; + pQuery->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; pQuery->limit.offset = 0; return; } - + if (QUERY_IS_ASC_QUERY(pQuery)) { pQuery->pos = pQuery->limit.offset; } else { @@ -4074,7 +4073,7 @@ static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBloc assert(pQuery->pos >= 0 && pQuery->pos <= pBlockInfo->rows - 1); - SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // update the pQuery->limit.offset value, and pQuery->pos value @@ -4083,10 +4082,10 @@ static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBloc // update the offset value pQuery->lastKey = keys[pQuery->pos]; pQuery->limit.offset = 0; - + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, - &pRuntimeEnv->windowResInfo, pDataBlock); - + &pRuntimeEnv->windowResInfo, pDataBlock); + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes); } @@ -4102,7 +4101,7 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; - + while (tsdbNextDataBlock(pQueryHandle)) { if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { return; @@ -4114,147 +4113,117 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= blockInfo.rows; pQuery->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; pQuery->lastKey += step; - - qTrace("QInfo:%p skip rows:%d, offset:%" PRId64 "", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows, pQuery->limit.offset); - } else { // find the appropriated start position in current block + + qTrace("QInfo:%p skip rows:%d, offset:%" PRId64 "", GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows, + pQuery->limit.offset); + } else { // find the appropriated start position in current block updateOffsetVal(pRuntimeEnv, &blockInfo); break; } } } -static UNUSED_FUNC bool forwardQueryStartPosIfNeeded(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; // if queried with value filter, do NOT forward query start position - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { + if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { return true; } - if (pQuery->limit.offset > 0 && (!isTopBottomQuery(pQuery)) && pQuery->interpoType == TSDB_INTERPO_NONE) { - /* - * 1. for top/bottom query, the offset applies to the final result, not here - * 2. for interval without interpolation query we forward pQuery->intervalTime at a time for - * pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is - * not valid. otherwise, we only forward pQuery->limit.offset number of points - */ - if (isIntervalQuery(pQuery)) { - assert(pRuntimeEnv->windowResInfo.prevSKey == 0); + /* + * 1. for interval without interpolation query we forward pQuery->intervalTime at a time for + * pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is + * not valid. otherwise, we only forward pQuery->limit.offset number of points + */ + assert(pRuntimeEnv->windowResInfo.prevSKey == 0); - TSKEY skey1, ekey1; - STimeWindow w = {0}; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + TSKEY skey1, ekey1; + STimeWindow w = {0}; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (!tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { - // todo handle no data situation + while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { + SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle); + + if (QUERY_IS_ASC_QUERY(pQuery) && pWindowResInfo->prevSKey == 0) { + getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, &ekey1, + &w); + pWindowResInfo->startTime = w.skey; + pWindowResInfo->prevSKey = w.skey; + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, + &w); + + pWindowResInfo->startTime = pQuery->window.skey; + pWindowResInfo->prevSKey = w.skey; + } + + // the first time window + STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery); + + while (pQuery->limit.offset > 0) { + if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + pQuery->limit.offset -= 1; + pWindowResInfo->prevSKey = win.skey; } - SDataBlockInfo blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle); + STimeWindow tw = win; + getNextTimeWindow(pQuery, &tw); - if (QUERY_IS_ASC_QUERY(pQuery)) { - getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &skey1, - &ekey1, &w); - pWindowResInfo->startTime = w.skey; - pWindowResInfo->prevSKey = w.skey; - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - TSKEY start = blockInfo.window.ekey - pQuery->intervalTime; - getAlignQueryTimeWindow(pQuery, start, pQuery->window.ekey, blockInfo.window.ekey, &skey1, &ekey1, &w); - - pWindowResInfo->startTime = pQuery->window.skey; - pWindowResInfo->prevSKey = w.skey; - } - - // the first time window - STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery); - - while (pQuery->limit.offset > 0) { - STimeWindow tw = win; - getNextTimeWindow(pQuery, &tw); - - // next time window starts from current data block + if (pQuery->limit.offset == 0) { if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { - // query completed - if ((tw.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (tw.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - setQueryStatus(pQuery, QUERY_COMPLETED); - break; - } - - tw = win; + // load the data block SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - int32_t startPos = getNextQualifiedWindow(pRuntimeEnv, &tw, pWindowResInfo, &blockInfo, pColInfoData->pData, - binarySearchForKey); + tw = win; + int32_t startPos = + getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey); assert(startPos >= 0); - pQuery->limit.offset -= 1; // set the abort info pQuery->pos = startPos; pQuery->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; pWindowResInfo->prevSKey = tw.skey; - win = tw; - continue; + + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, + &pRuntimeEnv->windowResInfo, pDataBlock); + + qTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", rows:%d, res:%d", + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes); + return true; } else { - if (!tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { - setQueryStatus(pQuery, QUERY_COMPLETED); - break; - } - - blockInfo = tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle); - if ((blockInfo.window.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (blockInfo.window.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - setQueryStatus(pQuery, QUERY_COMPLETED); - break; - } - - // set the window that start from the next data block - TSKEY key = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.skey : blockInfo.window.ekey; - STimeWindow n = getActiveTimeWindow(pWindowResInfo, key, pQuery); - - // next data block are still covered by current time window - if (n.skey == win.skey && n.ekey == win.ekey) { - // do nothing - } else { - pQuery->limit.offset -= 1; - - // query completed - if ((n.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (n.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - setQueryStatus(pQuery, QUERY_COMPLETED); - break; - } - - // set the abort info - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; - pQuery->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.skey : blockInfo.window.ekey; - pWindowResInfo->prevSKey = n.skey; - - win = n; - } + // do nothing, + return true; } } - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) || pQuery->limit.offset > 0) { - setQueryStatus(pQuery, QUERY_COMPLETED); - return false; + // next time window starts from current data block + if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + // load the data block, note that only the primary timestamp column is required + SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); + + tw = win; + int32_t startPos = + getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey); + assert(startPos >= 0); + + // set the abort info + pQuery->pos = startPos; + pQuery->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; + pWindowResInfo->prevSKey = tw.skey; + win = tw; } else { - assert(0); - // if (IS_DISK_DATA_BLOCK(pQuery)) { - // getTimestampInDiskBlock(pRuntimeEnv, 0); + break; // offset is not 0, and next time window locates in the next block. } } - } else { // forward the start position for projection query - skipBlocks(&pQInfo->runtimeEnv); - if (pQuery->limit.offset > 0) { - setQueryStatus(pQuery, QUERY_COMPLETED); - return false; - } } - + return true; } @@ -5006,7 +4975,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); } - + // skip blocks without load the actual data block from file if no filter condition present skipBlocks(&pQInfo->runtimeEnv); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { @@ -5090,6 +5059,13 @@ static void tableIntervalProcess(SQInfo *pQInfo) { int32_t numOfInterpo = 0; + // skip blocks without load the actual data block from file if no filter condition present + skipTimeInterval(pRuntimeEnv); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { + setQueryStatus(pQuery, QUERY_COMPLETED); + return; + } + while (1) { tableIntervalProcessImpl(pRuntimeEnv); From 64545fb9a3d12dc20aec5394effa550f23c124a0 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 22 Apr 2020 17:14:39 +0800 Subject: [PATCH 03/63] [td-171] add filter support, refactor the client side parse functions --- src/client/inc/tscUtil.h | 18 +- src/client/inc/tsclient.h | 24 +- src/client/src/tscFunctionImpl.c | 2 +- src/client/src/tscSQLParser.c | 42 ++- src/client/src/tscServer.c | 51 +-- src/client/src/tscSubquery.c | 50 +-- src/client/src/tscUtil.c | 247 ++++++-------- src/inc/taosmsg.h | 2 +- src/inc/tsdb.h | 2 +- src/query/inc/queryExecutor.h | 5 +- src/query/inc/queryUtil.h | 5 + src/query/src/queryExecutor.c | 262 +++++---------- src/query/src/queryFilterFunc.c | 558 +++++++++++++++++++++++++++++++ src/tsdb/src/tsdbRead.c | 12 +- src/util/inc/tarray.h | 4 +- src/util/src/tarray.c | 4 +- 16 files changed, 857 insertions(+), 431 deletions(-) create mode 100644 src/query/src/queryFilterFunc.c diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 57cf821eb1..70c2ed2c7a 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -57,7 +57,7 @@ typedef struct SJoinSubquerySupporter { int64_t interval; // interval time SLimitVal limit; // limit info uint64_t uid; // query meter uid - SColumnBaseInfo colList; // previous query information + SArray* colList; // previous query information SSqlExprInfo exprsInfo; SFieldInfo fieldsInfo; STagCond tagCond; @@ -106,7 +106,7 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); -bool tscQueryOnMetric(SSqlCmd* pCmd); +bool tscQueryOnSTable(SSqlCmd* pCmd); bool tscQueryTags(SQueryInfo* pQueryInfo); bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); @@ -159,16 +159,12 @@ void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t ui void* tscSqlExprDestroy(SSqlExpr* pExpr); void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo); -SColumnBase* tscColumnBaseInfoInsert(SQueryInfo* pQueryInfo, SColumnIndex* colIndex); -void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* src); -void tscColumnBaseCopy(SColumnBase* dst, const SColumnBase* src); +SColumn* tscColumnClone(const SColumn* src); +SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); +void tscColumnListAssign(SArray* dst, const SArray* src, int16_t tableIndex); +void tscColumnListDestroy(SArray* pColList); -void tscColumnBaseInfoCopy(SColumnBaseInfo* dst, const SColumnBaseInfo* src, int16_t tableIndex); -SColumnBase* tscColumnBaseInfoGet(SColumnBaseInfo* pColumnBaseInfo, int32_t index); -void tscColumnBaseInfoUpdateTableIndex(SColumnBaseInfo* pColList, int16_t tableIndex); - -void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size); -void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo); +SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); int32_t tscValidateName(SSQLToken* pToken); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index f225d546f3..38344c9c41 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -30,9 +30,9 @@ extern "C" { #include "taosmsg.h" #include "tarray.h" #include "tglobal.h" -#include "trpc.h" -#include "tsqlfunction.h" #include "tutil.h" +#include "tsqlfunction.h" +#include "queryExecutor.h" #define TSC_GET_RESPTR_BASE(res, _queryinfo, col) (res->data + ((_queryinfo)->fieldsInfo.pSqlExpr[col]->offset) * res->numOfRows) @@ -41,14 +41,6 @@ struct SSqlInfo; typedef SCMSTableVgroupRspMsg SVgroupsInfo; -typedef struct SSqlGroupbyExpr { - int16_t tableIndex; - int16_t numOfGroupCols; - SColIndex columnInfo[TSDB_MAX_TAGS]; // group by columns information - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc -} SSqlGroupbyExpr; - typedef struct STableComInfo { uint8_t numOfTags; uint8_t precision; @@ -123,17 +115,11 @@ typedef struct SSqlExprInfo { SSqlExpr** pExprs; } SSqlExprInfo; -typedef struct SColumnBase { +typedef struct SColumn { SColumnIndex colIndex; int32_t numOfFilters; SColumnFilterInfo *filterInfo; -} SColumnBase; - -typedef struct SColumnBaseInfo { - int16_t numOfAlloc; - int16_t numOfCols; - SColumnBase *pColList; -} SColumnBaseInfo; +} SColumn; struct SLocalReducer; @@ -223,7 +209,7 @@ typedef struct SQueryInfo { int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info - SColumnBaseInfo colList; + SArray* colList; SFieldInfo fieldsInfo; SSqlExprInfo exprsInfo; SLimitVal limit; diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 9a8bec4597..d3bf1c5206 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2940,7 +2940,7 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - pCtx->aOutputBuf += pCtx->inputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; + pCtx->aOutputBuf += pCtx->inputBytes; } /** diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c13acbba6a..cc9721b1fd 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1123,7 +1123,11 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel const char* msg5 = "invalid function name"; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - + + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + for (int32_t i = 0; i < pSelection->nExpr; ++i) { int32_t outputIndex = pQueryInfo->exprsInfo.numOfExprs; tSQLExprItem* pItem = &pSelection->a[i]; @@ -1278,7 +1282,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr) { for (int32_t i = 0; i < pIdList->num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(pIdList->ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(pIdList->ids[i])); } tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, outputIndex, type, fieldName, bytes); @@ -1494,7 +1498,7 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, // for all querie, the timestamp column meeds to be loaded SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnBaseInfoInsert(pQueryInfo, &index); + tscColumnListInsert(pQueryInfo->colList, &index); SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); insertResultField(pQueryInfo, resColIdx, &ids, bytes, type, columnName, pExpr); @@ -1579,12 +1583,12 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr insertResultField(pQueryInfo, numOfOutput, &ids, sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(ids.ids[i])); } } SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + tscColumnListInsert(pQueryInfo->colList, &tsCol); return TSDB_CODE_SUCCESS; } @@ -1690,12 +1694,12 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr insertResultField(pQueryInfo, numOfOutput, &ids, pExpr->resBytes, pExpr->resType, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(ids.ids[i])); } } SColumnIndex tsCol = {.tableIndex = index.tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnBaseInfoInsert(pQueryInfo, &tsCol); + tscColumnListInsert(pQueryInfo->colList, &tsCol); return TSDB_CODE_SUCCESS; } @@ -1889,7 +1893,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr insertResultField(pQueryInfo, colIndex, &ids, resultSize, resultType, pExpr->aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnBaseInfoInsert(pQueryInfo, &(ids.ids[i])); + tscColumnListInsert(pQueryInfo->colList, &(ids.ids[i])); } } @@ -2535,7 +2539,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return invalidSqlErrMsg(pQueryInfo->msg, msg8); } - tscColumnBaseInfoInsert(pQueryInfo, &index); + tscColumnListInsert(pQueryInfo->colList, &index); pQueryInfo->groupbyExpr.columnInfo[i] = (SColIndex){.colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId}; // relIndex; pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; @@ -2559,7 +2563,7 @@ void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo) { } } -static SColumnFilterInfo* addColumnFilterInfo(SColumnBase* pColumn) { +static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { if (pColumn == NULL) { return NULL; } @@ -2836,7 +2840,7 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn const char* msg1 = "non binary column not support like operator"; const char* msg2 = "binary column not support this operator"; - SColumnBase* pColumn = tscColumnBaseInfoInsert(pQueryInfo, pIndex); + SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex); SColumnFilterInfo* pColFilter = NULL; /* @@ -2857,10 +2861,10 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn return TSDB_CODE_INVALID_SQL; } - pColFilter->filterOnBinary = + pColFilter->filterstr = ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); - if (pColFilter->filterOnBinary) { + if (pColFilter->filterstr) { if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE && pExpr->nSQLOptr != TK_LIKE) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -3584,11 +3588,15 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac } static bool validateFilterExpr(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase* pColBase = &pQueryInfo->colList.pColList[i]; + SArray* pColList = pQueryInfo->colList; + + size_t num = taosArrayGetSize(pColList); + + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(pColList, i); - for (int32_t j = 0; j < pColBase->numOfFilters; ++j) { - SColumnFilterInfo* pColFilter = &pColBase->filterInfo[j]; + for (int32_t j = 0; j < pCol->numOfFilters; ++j) { + SColumnFilterInfo* pColFilter = &pCol->filterInfo[j]; int32_t lowerOptr = pColFilter->lowerRelOptr; int32_t upperOptr = pColFilter->upperRelOptr; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ad5f9cd0af..5c1a6dba0c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -567,7 +567,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); - int32_t srcColListSize = pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); + int32_t srcColListSize = taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo); int32_t exprSize = sizeof(SSqlFuncExprMsg) * pQueryInfo->exprsInfo.numOfExprs; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -624,7 +624,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (pQueryInfo->colList.numOfCols <= 0) { + if (taosArrayGetSize(pQueryInfo->colList) <= 0) { tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); return -1; } @@ -700,7 +700,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->interpoType = htons(pQueryInfo->interpoType); pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); - pQueryMsg->numOfCols = htons(pQueryInfo->colList.numOfCols); + pQueryMsg->numOfCols = htons(taosArrayGetSize(pQueryInfo->colList)); pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime); pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; @@ -716,21 +716,22 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } // set column list ids - char *pMsg = (char *)(pQueryMsg->colList) + pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + char *pMsg = (char *)(pQueryMsg->colList) + numOfCols * sizeof(SColumnInfo); SSchema *pSchema = tscGetTableSchema(pTableMeta); - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase *pCol = tscColumnBaseInfoGet(&pQueryInfo->colList, i); - SSchema * pColSchema = &pSchema[pCol->colIndex.columnIndex]; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); + SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; -// if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || -// pColSchema->type > TSDB_DATA_TYPE_NCHAR) { -// tscError("%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql, -// htons(pQueryMsg->vnode), pTableMeta->sid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex, -// pColSchema->name); -// -// return -1; // 0 means build msg failed -// } + if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || + pColSchema->type > TSDB_DATA_TYPE_NCHAR) { + tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", + pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex, + pColSchema->name); + + return -1; // 0 means build msg failed + } pQueryMsg->colList[i].colId = htons(pColSchema->colId); pQueryMsg->colList[i].bytes = htons(pColSchema->bytes); @@ -742,11 +743,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SColumnFilterInfo *pColFilter = &pCol->filterInfo[f]; SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; - pFilterMsg->filterOnBinary = htons(pColFilter->filterOnBinary); + pFilterMsg->filterstr = htons(pColFilter->filterstr); pMsg += sizeof(SColumnFilterInfo); - if (pColFilter->filterOnBinary) { + if (pColFilter->filterstr) { pFilterMsg->len = htobe64(pColFilter->len); memcpy(pMsg, (void *)pColFilter->pz, pColFilter->len + 1); pMsg += (pColFilter->len + 1); // append the additional filter binary info @@ -808,8 +809,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t len = 0; if (hasArithmeticFunction) { - SColumnBase *pColBase = pQueryInfo->colList.pColList; - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pColBase = taosArrayGetP(pQueryInfo->colList, i); + char * name = pSchema[pColBase[i].colIndex.columnIndex].name; int32_t lenx = strlen(name); memcpy(pMsg, name, lenx); @@ -2194,12 +2196,15 @@ int tscProcessShowRsp(SSqlObj *pSql) { pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscColumnBaseInfoReserve(&pQueryInfo->colList, pMetaMsg->numOfColumns); + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + SColumnIndex index = {0}; - for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i) { index.columnIndex = i; - tscColumnBaseInfoInsert(pQueryInfo, &index); + tscColumnListInsert(pQueryInfo->colList, &index); + tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pTableSchema[i]); pQueryInfo->fieldsInfo.pSqlExpr[i] = tscSqlExprInsert(pQueryInfo, i, TSDB_FUNC_TS_DUMMY, &index, @@ -2477,7 +2482,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *tableId) { * 1. only update the metermeta in force model metricmeta is not updated * 2. if get metermeta failed, still get the metermeta */ - if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnMetric(pCmd)) { + if (pTableMetaInfo->pTableMeta == NULL || !tscQueryOnSTable(pCmd)) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pTableMetaInfo->pTableMeta) { tscTrace("%p update table meta, old: numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e647772226..5b0bba4607 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -191,7 +191,7 @@ void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { } tscSqlExprInfoDestroy(&pSupporter->exprsInfo); - tscColumnBaseInfoDestroy(&pSupporter->colList); + tscColumnListDestroy(pSupporter->colList); tscClearFieldInfo(&pSupporter->fieldsInfo); @@ -211,8 +211,10 @@ void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { * */ bool needSecondaryQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->colList.numOfCols; ++i) { - SColumnBase* pBase = tscColumnBaseInfoGet(&pQueryInfo->colList, i); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pBase = taosArrayGet(pQueryInfo->colList, i); if (pBase->colIndex.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return true; } @@ -299,7 +301,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pQueryInfo->intervalTime = pSupporter->interval; pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; - tscColumnBaseInfoCopy(&pQueryInfo->colList, &pSupporter->colList, 0); + tscColumnListAssign(pQueryInfo->colList, pSupporter->colList, 0); tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); tscSqlExprCopy(&pQueryInfo->exprsInfo, &pSupporter->exprsInfo, pSupporter->uid, false); @@ -342,9 +344,10 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { tscPrintSelectClause(pNew, 0); + size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscTrace("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, + pNewQueryInfo->exprsInfo.numOfExprs, numOfCols, pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); } @@ -850,8 +853,14 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); assert(pNewQueryInfo != NULL); - tscColumnBaseInfoUpdateTableIndex(&pNewQueryInfo->colList, 0); - tscColumnBaseInfoCopy(&pSupporter->colList, &pNewQueryInfo->colList, 0); + // update the table index + size_t num = taosArrayGetSize(pNewQueryInfo->colList); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(pNewQueryInfo->colList, i); + pCol->colIndex.tableIndex = 0; + } + + tscColumnListAssign(pSupporter->colList, pNewQueryInfo->colList, 0); tscSqlExprCopy(&pSupporter->exprsInfo, &pNewQueryInfo->exprsInfo, pSupporter->uid, false); tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); @@ -888,27 +897,26 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu pExpr->numOfParams = 1; // add the filter tag column - for (int32_t i = 0; i < pSupporter->colList.numOfCols; ++i) { - SColumnBase *pColBase = &pSupporter->colList.pColList[i]; - if (pColBase->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. - tscColumnBaseCopy(&pNewQueryInfo->colList.pColList[pNewQueryInfo->colList.numOfCols], pColBase); - pNewQueryInfo->colList.numOfCols++; + size_t s = taosArrayGetSize(pSupporter->colList); + + for (int32_t i = 0; i < s; ++i) { + SColumn *pCol = taosArrayGetP(pSupporter->colList, i); + + if (pCol->numOfFilters > 0) { // copy to the pNew->cmd.colList if it is filtered. + SColumn* p = tscColumnClone(pCol); + taosArrayPush(pNewQueryInfo->colList, &p); } } - + + size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); + tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); - tscPrintSelectClause(pNew, 0); - - tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " - "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, pNewQueryInfo->colList.numOfCols, + pNewQueryInfo->exprsInfo.numOfExprs, numOfCols, pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); tscPrintSelectClause(pNew, 0); + } else { SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); pNewQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3650ea1f30..cb5e8c0d66 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -130,7 +130,7 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBuffer* pBuf) { taosArrayPush(pTagCond->pCond, &cond); } -bool tscQueryOnMetric(SSqlCmd* pCmd) { +bool tscQueryOnSTable(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); return ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) == TSDB_QUERY_TYPE_STABLE_QUERY) && @@ -1289,7 +1289,6 @@ void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo) { pExprInfo->numOfExprs = 0; } - void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t tableuid, bool deepcopy) { if (src == NULL || src->numOfExprs == 0) { return; @@ -1323,186 +1322,130 @@ void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t tableui } } } - } -static void clearVal(SColumnBase* pBase) { - memset(pBase, 0, sizeof(SColumnBase)); - - pBase->colIndex.tableIndex = -2; - pBase->colIndex.columnIndex = -2; -} - -static void _cf_ensureSpace(SColumnBaseInfo* pcolList, int32_t size) { - if (pcolList->numOfAlloc < size) { - int32_t oldSize = pcolList->numOfAlloc; - - int32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1); - while (newSize < size) { - newSize = (newSize << 1); - } - - if (newSize > TSDB_MAX_COLUMNS) { - newSize = TSDB_MAX_COLUMNS; - } - - int32_t inc = newSize - oldSize; - - pcolList->pColList = realloc(pcolList->pColList, newSize * sizeof(SColumnBase)); - memset(&pcolList->pColList[oldSize], 0, inc * sizeof(SColumnBase)); - - pcolList->numOfAlloc = newSize; - } -} - -static void _cf_evic(SColumnBaseInfo* pcolList, int32_t index) { - if (index < pcolList->numOfCols) { - memmove(&pcolList->pColList[index + 1], &pcolList->pColList[index], - sizeof(SColumnBase) * (pcolList->numOfCols - index)); - - clearVal(&pcolList->pColList[index]); - } -} - -SColumnBase* tscColumnBaseInfoGet(SColumnBaseInfo* pColumnBaseInfo, int32_t index) { - if (pColumnBaseInfo == NULL || pColumnBaseInfo->numOfCols < index) { - return NULL; - } - - return &pColumnBaseInfo->pColList[index]; -} - -void tscColumnBaseInfoUpdateTableIndex(SColumnBaseInfo* pColList, int16_t tableIndex) { - for (int32_t i = 0; i < pColList->numOfCols; ++i) { - pColList->pColList[i].colIndex.tableIndex = tableIndex; - } -} - -// todo refactor -SColumnBase* tscColumnBaseInfoInsert(SQueryInfo* pQueryInfo, SColumnIndex* pColIndex) { - SColumnBaseInfo* pcolList = &pQueryInfo->colList; - +SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { // ignore the tbname column to be inserted into source list if (pColIndex->columnIndex < 0) { return NULL; } - + + size_t numOfCols = taosArrayGetSize(pColumnList); int16_t col = pColIndex->columnIndex; int32_t i = 0; - while (i < pcolList->numOfCols) { - if (pcolList->pColList[i].colIndex.columnIndex < col) { + while (i < numOfCols) { + SColumn* pCol = taosArrayGetP(pColumnList, i); + if (pCol->colIndex.columnIndex < col) { i++; - } else if (pcolList->pColList[i].colIndex.tableIndex < pColIndex->tableIndex) { + } else if (pCol->colIndex.tableIndex < pColIndex->tableIndex) { i++; } else { break; } } - SColumnIndex* pIndex = &pcolList->pColList[i].colIndex; - if ((i < pcolList->numOfCols && (pIndex->columnIndex > col || pIndex->tableIndex != pColIndex->tableIndex)) || - (i >= pcolList->numOfCols)) { - _cf_ensureSpace(pcolList, pcolList->numOfCols + 1); - _cf_evic(pcolList, i); - - pcolList->pColList[i].colIndex = *pColIndex; - pcolList->numOfCols++; - } - - return &pcolList->pColList[i]; -} - -void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* src) { - assert(src != NULL && dst != NULL); - - assert(src->filterOnBinary == 0 || src->filterOnBinary == 1); - if (src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID) { - assert(0); - } - - *dst = *src; - if (dst->filterOnBinary) { - size_t len = (size_t)dst->len + 1; - char* pTmp = calloc(1, len); - dst->pz = (int64_t)pTmp; - memcpy((char*)dst->pz, (char*)src->pz, (size_t)len); - } -} - -void tscColumnBaseCopy(SColumnBase* dst, const SColumnBase* src) { - assert(src != NULL && dst != NULL); - - *dst = *src; - - if (src->numOfFilters > 0) { - dst->filterInfo = calloc(1, src->numOfFilters * sizeof(SColumnFilterInfo)); - - for (int32_t j = 0; j < src->numOfFilters; ++j) { - tscColumnFilterInfoCopy(&dst->filterInfo[j], &src->filterInfo[j]); - } + if (i >= numOfCols || numOfCols == 0) { + SColumn* b = calloc(1, sizeof(SColumn)); + b->colIndex = *pColIndex; + + taosArrayInsert(pColumnList, i, &b); } else { - assert(src->filterInfo == NULL); + SColumn* pCol = taosArrayGetP(pColumnList, i); + + if (i < numOfCols && (pCol->colIndex.columnIndex > col || pCol->colIndex.tableIndex != pColIndex->tableIndex)) { + SColumn* b = calloc(1, sizeof(SColumn)); + b->colIndex = *pColIndex; + + taosArrayInsert(pColumnList, i, &b); + } } + + return taosArrayGetP(pColumnList, i); } -void tscColumnBaseInfoCopy(SColumnBaseInfo* dst, const SColumnBaseInfo* src, int16_t tableIndex) { +SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) { + SColumnFilterInfo* pFilter = NULL; + if (numOfFilters > 0) { + pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); + } else { + assert(src == NULL); + return NULL; + } + + memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); + for (int32_t j = 0; j < numOfFilters; ++j) { + if (pFilter[j].filterstr) { + size_t len = (size_t) pFilter[j].len + 1; + + char* pTmp = calloc(1, len); + pFilter[j].pz = (int64_t) pTmp; + + memcpy((char*)pFilter[j].pz, (char*)src->pz, (size_t)len); + } + } + + assert(src->filterstr == 0 || src->filterstr == 1); + assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID)); + + return pFilter; +} + +static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) { + for(int32_t i = 0; i < numOfFilters; ++i) { + if (pFilterInfo[i].filterstr) { + tfree(pFilterInfo[i].pz); + } + } + + tfree(pFilterInfo); +} + +SColumn* tscColumnClone(const SColumn* src) { + assert(src != NULL); + + SColumn* dst = calloc(1, sizeof(SColumn)); + + dst->colIndex = src->colIndex; + dst->numOfFilters = src->numOfFilters; + dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); + + return dst; +} + +static void tscColumnDestroy(SColumn* pCol) { + destroyFilterInfo(pCol->filterInfo, pCol->numOfFilters); + free(pCol); +} + +void tscColumnListAssign(SArray* dst, const SArray* src, int16_t tableIndex) { if (src == NULL) { return; } + + size_t num = taosArrayGetSize(src); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(src, i); - *dst = *src; - dst->pColList = calloc(1, sizeof(SColumnBase) * dst->numOfAlloc); - - int16_t num = 0; - for (int32_t i = 0; i < src->numOfCols; ++i) { - if (src->pColList[i].colIndex.tableIndex == tableIndex || tableIndex < 0) { - dst->pColList[num] = src->pColList[i]; - - if (dst->pColList[num].numOfFilters > 0) { - dst->pColList[num].filterInfo = calloc(1, dst->pColList[num].numOfFilters * sizeof(SColumnFilterInfo)); - - for (int32_t j = 0; j < dst->pColList[num].numOfFilters; ++j) { - tscColumnFilterInfoCopy(&dst->pColList[num].filterInfo[j], &src->pColList[i].filterInfo[j]); - } - } - - num += 1; + if (pCol->colIndex.tableIndex == tableIndex || tableIndex < 0) { + SColumn* p = tscColumnClone(pCol); + taosArrayPush(dst, &p); } } - - dst->numOfCols = num; } -void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo) { +void tscColumnListDestroy(SArray* pColumnBaseInfo) { if (pColumnBaseInfo == NULL) { return; } - assert(pColumnBaseInfo->numOfCols <= TSDB_MAX_COLUMNS); - - for (int32_t i = 0; i < pColumnBaseInfo->numOfCols; ++i) { - SColumnBase* pColBase = &(pColumnBaseInfo->pColList[i]); - - if (pColBase->numOfFilters > 0) { - for (int32_t j = 0; j < pColBase->numOfFilters; ++j) { - assert(pColBase->filterInfo[j].filterOnBinary == 0 || pColBase->filterInfo[j].filterOnBinary == 1); - - if (pColBase->filterInfo[j].filterOnBinary) { - free((char*)pColBase->filterInfo[j].pz); - pColBase->filterInfo[j].pz = 0; - } - } - } - - tfree(pColBase->filterInfo); + size_t num = taosArrayGetSize(pColumnBaseInfo); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGetP(pColumnBaseInfo, i); + tscColumnDestroy(pCol); } - tfree(pColumnBaseInfo->pColList); -} - -void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size) { - _cf_ensureSpace(pColumnBaseInfo, size); + taosArrayDestroy(pColumnBaseInfo); } /* @@ -1883,7 +1826,7 @@ static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { tscSqlExprInfoDestroy(&pQueryInfo->exprsInfo); memset(&pQueryInfo->exprsInfo, 0, sizeof(pQueryInfo->exprsInfo)); - tscColumnBaseInfoDestroy(&pQueryInfo->colList); + tscColumnListDestroy(pQueryInfo->colList); memset(&pQueryInfo->colList, 0, sizeof(pQueryInfo->colList)); pQueryInfo->tsBuf = tsBufDestory(pQueryInfo->tsBuf); @@ -2070,7 +2013,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void return NULL; } - tscColumnBaseInfoCopy(&pNewQueryInfo->colList, &pQueryInfo->colList, (int16_t)tableIndex); + tscColumnListAssign(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); // set the correct query type if (pPrevSql != NULL) { @@ -2149,11 +2092,13 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } if (cmd == TSDB_SQL_SELECT) { + size_t size = taosArrayGetSize(pNewQueryInfo->colList); + tscTrace( "%p new subquery: %p, tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, pNewQueryInfo->exprsInfo.numOfExprs, - pNewQueryInfo->colList.numOfCols, pNewQueryInfo->fieldsInfo.numOfOutputCols, pFinalInfo->name, pNewQueryInfo->stime, + size, pNewQueryInfo->fieldsInfo.numOfOutputCols, pFinalInfo->name, pNewQueryInfo->stime, pNewQueryInfo->etime, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); tscPrintSelectClause(pNew, 0); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index f292345e68..d460d100a0 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -413,7 +413,7 @@ typedef struct SSqlFunctionExpr { typedef struct SColumnFilterInfo { int16_t lowerRelOptr; int16_t upperRelOptr; - int16_t filterOnBinary; /* denote if current column is binary */ + int16_t filterstr; // denote if current column is char(binary/nchar) union { struct { diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 3e15257fcb..8d33e0bbf7 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -146,7 +146,7 @@ typedef struct STsdbQueryCond { STimeWindow twindow; int32_t order; // desc/asc order to iterate the data block int32_t numOfCols; - SColumnInfoData *colList; + SColumnInfo *colList; } STsdbQueryCond; typedef struct SBlockInfo { diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index c07897abf6..504d20f992 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -90,7 +90,7 @@ typedef struct SColumnFilterElem { } SColumnFilterElem; typedef struct SSingleColumnFilterInfo { - SColumnInfoData info; + SColumnInfo info; int32_t numOfFilters; SColumnFilterElem* pFilters; void* pData; @@ -129,14 +129,13 @@ typedef struct SQuery { int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; SSqlFunctionExpr* pSelectExpr; - SColumnInfoData* colList; + SColumnInfo* colList; int32_t numOfFilterCols; int64_t* defaultVal; TSKEY lastKey; uint32_t status; // query status SResultRec rec; int32_t pos; - int64_t pointsOffset; // the number of points offset to save read data SData** sdata; SSingleColumnFilterInfo* pFilterInfo; } SQuery; diff --git a/src/query/inc/queryUtil.h b/src/query/inc/queryUtil.h index 677ddf6378..e21504b908 100644 --- a/src/query/inc/queryUtil.h +++ b/src/query/inc/queryUtil.h @@ -38,4 +38,9 @@ void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTa char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult); +__filter_func_t *getRangeFilterFuncArray(int32_t type); +__filter_func_t *getValueFilterFuncArray(int32_t type); + +bool supportPrefilter(int32_t type); + #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index c1208dbc0b..aefe4a3b8b 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -27,6 +27,7 @@ #include "tscompression.h" #include "tsdbMain.h" //todo use TableId instead of STable object #include "ttime.h" +#include "tscUtil.h" // todo move the function to common module #define DEFAULT_INTERN_BUF_SIZE 16384L @@ -52,8 +53,8 @@ /* get the qinfo struct address from the query struct address */ #define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.bytes) -#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].info.type) + ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].bytes) +#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].type) typedef struct SPointInterpoSupporter { int32_t numOfCols; @@ -223,19 +224,19 @@ bool getNeighborPoints(SQInfo *pQInfo, void *pMeterObj, SPointInterpoSupporter * return true; } -bool vnodeDoFilterData(SQuery *pQuery, int32_t elemPos) { +bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - char * pElem = pFilterInfo->pData + pFilterInfo->info.info.bytes * elemPos; - - if (isNull(pElem, pFilterInfo->info.info.type)) { + + char *pElem = pFilterInfo->pData + pFilterInfo->info.bytes * elemPos; + if (isNull(pElem, pFilterInfo->info.type)) { return false; } - int32_t num = pFilterInfo->numOfFilters; - bool qualified = false; - for (int32_t j = 0; j < num; ++j) { + bool qualified = false; + for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j]; + if (pFilterElem->fp(pFilterElem, pElem, pElem)) { qualified = true; break; @@ -252,7 +253,7 @@ bool vnodeDoFilterData(SQuery *pQuery, int32_t elemPos) { bool vnodeFilterData(SQuery *pQuery, int32_t *numOfActualRead, int32_t index) { (*numOfActualRead)++; - if (!vnodeDoFilterData(pQuery, index)) { + if (!doFilterData(pQuery, index)) { return false; } @@ -333,8 +334,8 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { } for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (colId == pQuery->colList[i].info.colId) { - type = pQuery->colList[i].info.type; + if (colId == pQuery->colList[i].colId) { + type = pQuery->colList[i].type; break; } } @@ -367,7 +368,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS_COMP; } -bool doRevisedResultsByLimit(SQInfo *pQInfo) { +bool limitResults(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { @@ -848,10 +849,9 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 } for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - SColumnInfo *pColMsg = &pQuery->colList[i].info; + SColumnInfo *pColMsg = &pQuery->colList[i]; assert(0); - // char * pData = doGetDataBlocks(pQuery, pRuntimeEnv->colDataBuffer, pQuery->colList[i].colIdxInBuf); - +// char * pData = doGetDataBlocks(pQuery, pRuntimeEnv->colDataBuffer, pQuery->colList[i].colIdxInBuf); sas->elemSize[i] = pColMsg->bytes; // sas->data[i] = pData + pCtx->startOffset * sas->elemSize[i]; // start from the offset } @@ -860,7 +860,7 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 sas->offset = 0; } else { // other type of query function SColIndex *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; - if (TSDB_COL_IS_TAG(pCol->flag)) { + if (TSDB_COL_IS_TAG(pCol->flag) || pDataBlock == NULL) { dataBlock = NULL; } else { /* @@ -868,11 +868,8 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * stage, the remain meter may not have the required column in cache actually. So, the validation of required * column in cache with the corresponding meter schema is reinforced. */ - if (pDataBlock == NULL) { - return NULL; - } - int32_t numOfCols = taosArrayGetSize(pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (pCol->colId == p->info.colId) { @@ -896,7 +893,7 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 * @return the incremental number of output value, so it maybe 0 for fixed number of query, * such as count/min/max etc. */ -static void blockwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, +static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -1021,7 +1018,7 @@ static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int1 int32_t colId = pGroupbyExpr->columnInfo[k].colId; for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].info.colId == colId) { + if (pQuery->colList[i].colId == colId) { colIndex = i; break; } @@ -1029,8 +1026,8 @@ static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int1 assert(colIndex >= 0 && colIndex < pQuery->numOfCols); - *type = pQuery->colList[colIndex].info.type; - *bytes = pQuery->colList[colIndex].info.bytes; + *type = pQuery->colList[colIndex].type; + *bytes = pQuery->colList[colIndex].bytes; // groupbyColumnData = doGetDataBlocks(pQuery, data, pQuery->colList[colIndex].inf); break; @@ -1093,22 +1090,14 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return true; } -static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, - SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, - SArray *pDataBlock) { +static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, + SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - SQuery * pQuery = pRuntimeEnv->pQuery; - TSKEY * primaryKeyCol = (TSKEY *)taosArrayGet(pDataBlock, 0); - - // SData **data = pRuntimeEnv->colDataBuffer; - - int64_t prevNumOfRes = 0; - bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); - - if (!groupbyStateValue) { - prevNumOfRes = getNumOfResult(pRuntimeEnv); - } + + SQuery *pQuery = pRuntimeEnv->pQuery; + TSKEY *primaryKeyCol = (TSKEY*) ((SColumnInfoData *)taosArrayGet(pDataBlock, 0))->pData; + bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); int16_t type = 0; @@ -1134,16 +1123,10 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat // set the input column data for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - // SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - assert(0); - /* - * NOTE: here the tbname/tags column cannot reach here, since it will never be a filter column, - * so we do NOT check if is a tag or not - */ - // pFilterInfo->pData = doGetDataBlocks(pQuery, data, pFilterInfo->info.colIdxInBuf); + SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; + pFilterInfo->pData = getDataBlocks(pRuntimeEnv, &sasArray[k], pFilterInfo->info.colId, pDataBlockInfo->rows, pDataBlock); } - int32_t numOfRes = 0; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // from top to bottom in desc @@ -1171,7 +1154,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat } } - if (pQuery->numOfFilterCols > 0 && (!vnodeDoFilterData(pQuery, offset))) { + if (pQuery->numOfFilterCols > 0 && (!doFilterData(pQuery, offset))) { continue; } @@ -1181,9 +1164,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat int64_t ts = primaryKeyCol[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - assert(0); - int32_t ret = 0; - // int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pRuntimeEnv->pTabObj->sid, &win); + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &win); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1197,8 +1178,6 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat lastKey = ts; STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; - assert(0); - int32_t sid = 0; // pRuntimeEnv->pTabObj->sid; while (1) { getNextTimeWindow(pQuery, &nextWin); @@ -1213,7 +1192,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat } // null data, failed to allocate more memory buffer - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, sid, &nextWin) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->sid, &nextWin) != TSDB_CODE_SUCCESS) { break; } @@ -1254,54 +1233,10 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStat break; } } - - /* - * pointsOffset is the maximum available space in result buffer update the actual forward step for query that - * requires checking buffer during loop - */ - if ((pQuery->checkBuffer == 1) && (++numOfRes) >= pQuery->pointsOffset) { - pQuery->lastKey = lastKey + step; - assert(0); - // *forwardStep = j + 1; - break; - } } - + + pQuery->lastKey = lastKey + step; free(sasArray); - - /* - * No need to calculate the number of output results for group-by normal columns, interval query - * because the results of group by normal column is put into intermediate buffer. - */ - int32_t num = 0; - if (!groupbyStateValue && !isIntervalQuery(pQuery)) { - num = getNumOfResult(pRuntimeEnv) - prevNumOfRes; - } - - return num; -} - -static UNUSED_FUNC int32_t reviseForwardSteps(SQueryRuntimeEnv *pRuntimeEnv, int32_t forwardStep) { - /* - * 1. If value filter exists, we try all data in current block, and do not set the QUERY_RESBUF_FULL flag. - * - * 2. In case of top/bottom/ts_comp query, the checkBuffer == 1 and pQuery->numOfFilterCols - * may be 0 or not. We do not check the capacity of output buffer, since the filter function will do it. - * - * 3. In handling the query of secondary query of join, tsBuf servers as a ts filter. - */ - SQuery *pQuery = pRuntimeEnv->pQuery; - - if (isTopBottomQuery(pQuery) || isTSCompQuery(pQuery) || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { - return forwardStep; - } - - // current buffer does not have enough space, try in the next loop - if ((pQuery->checkBuffer == 1) && (pQuery->pointsOffset <= forwardStep)) { - forwardStep = pQuery->pointsOffset; - } - - return forwardStep; } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, @@ -1310,9 +1245,9 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl SQuery *pQuery = pRuntimeEnv->pQuery; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - /*numOfRes = */ rowwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { - blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; @@ -1854,7 +1789,7 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { bool vnodeParametersSafetyCheck(SQuery *pQuery) { // load data column information is incorrect for (int32_t i = 0; i < pQuery->numOfCols - 1; ++i) { - if (pQuery->colList[i].info.colId == pQuery->colList[i + 1].info.colId) { + if (pQuery->colList[i].colId == pQuery->colList[i + 1].colId) { qError("QInfo:%p invalid data load column for query", GET_QINFO_ADDR(pQuery)); return false; } @@ -2133,7 +2068,7 @@ void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSu /* get appropriated size for one row data source*/ int32_t len = 0; for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - len += pQuery->colList[i].info.bytes; + len += pQuery->colList[i].bytes; } // assert(PRIMARY_TSCOL_LOADED(pQuery)); @@ -2147,7 +2082,7 @@ void pointInterpSupporterInit(SQuery *pQuery, SPointInterpoSupporter *pInterpoSu pInterpoSupport->pPrevPoint[i] = prev + offset; pInterpoSupport->pNextPoint[i] = next + offset; - offset += pQuery->colList[i].info.bytes; + offset += pQuery->colList[i].bytes; } } } @@ -3252,7 +3187,7 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { } } -void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { +void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; if (pQuery->rec.rows == 0 || pQuery->limit.offset == 0) { return; @@ -3260,23 +3195,21 @@ void doSkipResults(SQueryRuntimeEnv *pRuntimeEnv) { if (pQuery->rec.rows <= pQuery->limit.offset) { pQuery->limit.offset -= pQuery->rec.rows; - pQuery->rec.rows = 0; - // pQuery->pointsOffset = pQuery->rec.pointsToRead; // clear all data in result buffer resetCtxOutputBuf(pRuntimeEnv); // clear the buffer is full flag if exists pQuery->status &= (~QUERY_RESBUF_FULL); } else { - int32_t numOfSkip = (int32_t)pQuery->limit.offset; + int32_t numOfSkip = (int32_t) pQuery->limit.offset; pQuery->rec.rows -= numOfSkip; for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - assert(0); - // memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->size * bytes); + + memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->rec.rows * bytes); pRuntimeEnv->pCtx[i].aOutputBuf += bytes * numOfSkip; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { @@ -3864,9 +3797,9 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo * SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { - // numOfRes = rowwiseApplyAllFunctions(pRuntimeEnv, &forwardStep, pFields, pDataBlockInfo, pWindowResInfo); + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { - blockwiseApplyAllFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); } updateWindowResNumOfRes(pRuntimeEnv, pTableDataInfo); @@ -4667,10 +4600,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { scanAllDataBlocks(pRuntimeEnv); pQuery->rec.rows = getNumOfResult(pRuntimeEnv); - doSkipResults(pRuntimeEnv); + skipResults(pRuntimeEnv); // the limitation of output result is reached, set the query completed - if (doRevisedResultsByLimit(pQInfo)) { + if (limitResults(pQInfo)) { pQInfo->tableIndex = pQInfo->groupInfo.numOfTables; break; } @@ -4963,8 +4896,8 @@ static void tableFixedOutputProcess(SQInfo *pQInfo) { assert(isTopBottomQuery(pQuery)); } - doSkipResults(pRuntimeEnv); - doRevisedResultsByLimit(pQInfo); + skipResults(pRuntimeEnv); + limitResults(pQInfo); } static void tableMultiOutputProcess(SQInfo *pQInfo) { @@ -4993,7 +4926,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { pQuery->rec.rows = getNumOfResult(pRuntimeEnv); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.rows > 0) { - doSkipResults(pRuntimeEnv); + skipResults(pRuntimeEnv); } /* @@ -5010,7 +4943,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo) { resetCtxOutputBuf(pRuntimeEnv); } - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { qTrace("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->lastKey, pQuery->window.ekey); @@ -5079,7 +5012,7 @@ static void tableIntervalProcess(SQInfo *pQInfo) { // the offset is handled at prepare stage if no interpolation involved if (pQuery->interpoType == TSDB_INTERPO_NONE) { - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); break; } else { taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.rows, pQuery->interpoType); @@ -5095,7 +5028,7 @@ static void tableIntervalProcess(SQInfo *pQInfo) { qTrace("QInfo: %p interpo completed, final:%d", pQInfo, pQuery->rec.rows); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); break; } @@ -5129,7 +5062,7 @@ static void tableQueryImpl(SQInfo *pQInfo) { pQuery->rec.rows = vnodeQueryResultInterpolate(pQInfo, (tFilePage **)pQuery->sdata, (tFilePage **)pRuntimeEnv->pInterpoBuf, remain, &numOfInterpo); - doRevisedResultsByLimit(pQInfo); + limitResults(pQInfo); pQInfo->pointsInterpo += numOfInterpo; qTrace("QInfo:%p current:%d returned, total:%d", pQInfo, pQuery->rec.rows, pQuery->rec.total); @@ -5361,11 +5294,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; SColumnFilterInfo *pDestFilterInfo = &pColInfo->filters[f]; - pDestFilterInfo->filterOnBinary = htons(pFilterInfo->filterOnBinary); + pDestFilterInfo->filterstr = htons(pFilterInfo->filterstr); pMsg += sizeof(SColumnFilterInfo); - if (pDestFilterInfo->filterOnBinary) { + if (pDestFilterInfo->filterstr) { pDestFilterInfo->len = htobe64(pFilterInfo->len); pDestFilterInfo->pz = (int64_t)calloc(1, pDestFilterInfo->len + 1); @@ -5635,9 +5568,9 @@ static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SCol return pGroupbyExpr; } -static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { +static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].info.numOfFilters > 0) { + if (pQuery->colList[i].numOfFilters > 0) { pQuery->numOfFilterCols++; } } @@ -5649,18 +5582,18 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols); for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].info.numOfFilters > 0) { + if (pQuery->colList[i].numOfFilters > 0) { SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[j]; memcpy(&pFilterInfo->info, &pQuery->colList[i], sizeof(SColumnInfoData)); - pFilterInfo->info.info.filters = NULL; - - pFilterInfo->numOfFilters = pQuery->colList[i].info.numOfFilters; + pFilterInfo->info = pQuery->colList[i]; + + pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters; pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem)); for (int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) { SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f]; - pSingleColFilter->filterInfo = pQuery->colList[i].info.filters[f]; + pSingleColFilter->filterInfo = pQuery->colList[i].filters[f]; int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; @@ -5670,11 +5603,12 @@ static int32_t vnodeCreateFilterInfo(void *pQInfo, SQuery *pQuery) { return TSDB_CODE_INVALID_QUERY_MSG; } - int16_t type = pQuery->colList[i].info.type; - int16_t bytes = pQuery->colList[i].info.bytes; + int16_t type = pQuery->colList[i].type; + int16_t bytes = pQuery->colList[i].bytes; - __filter_func_t *rangeFilterArray = NULL; // vnodeGetRangeFilterFuncArray(type); - __filter_func_t *filterArray = NULL; // vnodeGetValueFilterFuncArray(type); + // todo refactor + __filter_func_t *rangeFilterArray = getRangeFilterFuncArray(type); + __filter_func_t *filterArray = getValueFilterFuncArray(type); if (rangeFilterArray == NULL && filterArray == NULL) { qError("QInfo:%p failed to get filter function, invalid data type:%d", pQInfo, type); @@ -5744,7 +5678,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { SColIndex *pColIndexEx = &pSqlExprMsg->colInfo; for (int32_t f = 0; f < pQuery->numOfCols; ++f) { - if (pColIndexEx->colId == pQuery->colList[f].info.colId) { + if (pColIndexEx->colId == pQuery->colList[f].colId) { pColIndexEx->colIndex = f; break; } @@ -5765,44 +5699,29 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutputCols = pQueryMsg->numOfOutputCols; - pQuery->numOfCols = numOfCols; + pQuery->numOfCols = numOfCols; pQuery->numOfOutputCols = numOfOutputCols; - - pQuery->limit.limit = pQueryMsg->limit; - pQuery->limit.offset = pQueryMsg->offset; - - pQuery->order.order = pQueryMsg->order; + pQuery->limit.limit = pQueryMsg->limit; + pQuery->limit.offset = pQueryMsg->offset; + pQuery->order.order = pQueryMsg->order; pQuery->order.orderColId = pQueryMsg->orderColId; - - pQuery->pSelectExpr = pExprs; - pQuery->pGroupbyExpr = pGroupbyExpr; - - pQuery->intervalTime = pQueryMsg->intervalTime; - - pQuery->slidingTime = pQueryMsg->slidingTime; + pQuery->pSelectExpr = pExprs; + pQuery->pGroupbyExpr = pGroupbyExpr; + pQuery->intervalTime = pQueryMsg->intervalTime; + pQuery->slidingTime = pQueryMsg->slidingTime; pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit; - - pQuery->interpoType = pQueryMsg->interpoType; + pQuery->interpoType = pQueryMsg->interpoType; pQuery->colList = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfCols); if (pQuery->colList == NULL) { - goto _clean_memory; + goto _cleanup; } for (int16_t i = 0; i < numOfCols; ++i) { - pQuery->colList[i].info = pQueryMsg->colList[i]; + pQuery->colList[i] = pQueryMsg->colList[i]; - SColumnInfo *pColInfo = &pQuery->colList[i].info; - pColInfo->filters = NULL; - // if (colList[i].numOfFilters > 0) { - // pColInfo->filters = calloc(1, colList[i].numOfFilters * sizeof(SColumnFilterInfo)); - // - // for (int32_t j = 0; j < colList[i].numOfFilters; ++j) { - // tscColumnFilterInfoCopy(&pColInfo->filters[j], &colList[i].filters[j]); - // } - // } else { - // pQuery->colList[i].data.filters = NULL; - // } + SColumnInfo *pColInfo = &pQuery->colList[i]; + pColInfo->filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pColInfo->numOfFilters); } // calculate the result row size @@ -5813,15 +5732,15 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou doUpdateExprColumnIndex(pQuery); - int32_t ret = vnodeCreateFilterInfo(pQInfo, pQuery); + int32_t ret = createFilterInfo(pQInfo, pQuery); if (ret != TSDB_CODE_SUCCESS) { - goto _clean_memory; + goto _cleanup; } // prepare the result buffer pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, POINTER_BYTES); if (pQuery->sdata == NULL) { - goto _clean_memory; + goto _cleanup; } // set the output buffer capacity @@ -5835,14 +5754,14 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou size_t size = (pQuery->rec.capacity + 1) * pExprs[col].resBytes + pExprs[col].interResBytes + sizeof(SData); pQuery->sdata[col] = (SData *)calloc(1, size); if (pQuery->sdata[col] == NULL) { - goto _clean_memory; + goto _cleanup; } } if (pQuery->interpoType != TSDB_INTERPO_NONE) { pQuery->defaultVal = malloc(sizeof(int64_t) * pQuery->numOfOutputCols); if (pQuery->defaultVal == NULL) { - goto _clean_memory; + goto _cleanup; } // the first column is the timestamp @@ -5861,7 +5780,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, strerror(errno)); - goto _clean_memory; + goto _cleanup; } vnodeParametersSafetyCheck(pQuery); @@ -5869,7 +5788,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou qTrace("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo); return pQInfo; -_clean_memory: +_cleanup: tfree(pQuery->defaultVal); if (pQuery->sdata != NULL) { @@ -6082,7 +6001,6 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) goto _query_over; } - // todo check vnode status if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); code = TSDB_CODE_INVALID_QUERY_MSG; diff --git a/src/query/src/queryFilterFunc.c b/src/query/src/queryFilterFunc.c new file mode 100644 index 0000000000..3218b26179 --- /dev/null +++ b/src/query/src/queryFilterFunc.c @@ -0,0 +1,558 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#include "taosmsg.h" +#include "tsqlfunction.h" +#include "queryExecutor.h" +#include "tcompare.h" + +bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval < pFilter->filterInfo.upperBndi); +} + +bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval < pFilter->filterInfo.upperBndd); +} + +bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval < pFilter->filterInfo.upperBndd); +} + +////////////////////////////////////////////////////////////////// +bool large_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +bool large_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)maxval > pFilter->filterInfo.lowerBndd); +} + +bool large_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)maxval > pFilter->filterInfo.lowerBndd); +} +///////////////////////////////////////////////////////////////////// + +bool lessEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi); +} + +bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd); +} + +bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd); +} + +////////////////////////////////////////////////////////////////////////// +bool largeEqual_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +//////////////////////////////////////////////////////////////////////// + +bool equal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int8_t *)minval == *(int8_t *)maxval) { + return (*(int8_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int8_t *)minval < *(int8_t *)maxval); + + return *(int8_t *)minval <= pFilter->filterInfo.lowerBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int16_t *)minval == *(int16_t *)maxval) { + return (*(int16_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int16_t *)minval < *(int16_t *)maxval); + + return *(int16_t *)minval <= pFilter->filterInfo.lowerBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int32_t *)minval == *(int32_t *)maxval) { + return (*(int32_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int32_t *)minval < *(int32_t *)maxval); + + return *(int32_t *)minval <= pFilter->filterInfo.lowerBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int64_t *)minval == *(int64_t *)maxval) { + return (*(int64_t *)minval == pFilter->filterInfo.lowerBndi); + } else { /* range filter */ + assert(*(int64_t *)minval < *(int64_t *)maxval); + + return *(int64_t *)minval <= pFilter->filterInfo.lowerBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(float *)minval == *(float *)maxval) { + return (fabs(*(float *)minval - pFilter->filterInfo.lowerBndd) <= FLT_EPSILON); + } else { /* range filter */ + assert(*(float *)minval < *(float *)maxval); + return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd; + } +} + +bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(double *)minval == *(double *)maxval) { + return (*(double *)minval == pFilter->filterInfo.lowerBndd); + } else { /* range filter */ + assert(*(double *)minval < *(double *)maxval); + + return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi; + } +} + +bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + // query condition string is greater than the max length of string, not qualified data + if (pFilter->filterInfo.len > pFilter->bytes) { + return false; + } + + return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0; +} + +bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) { + // query condition string is greater than the max length of string, not qualified data + if (pFilter->filterInfo.len > pFilter->bytes) { + return false; + } + + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0; +} + +//////////////////////////////////////////////////////////////// +bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH; +} + +bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) { + SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; + + return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; +} + +//////////////////////////////////////////////////////////////// +/** + * If minval equals to maxval, it may serve as the one element filter, + * or all elements of an array are identical during pref-filter stage. + * Otherwise, it must be pre-filter of array list of elements. + * + * During pre-filter stage, if there is one element that locates in [minval, maxval], + * the filter function will return true. + */ +bool nequal_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int8_t *)minval == *(int8_t *)maxval) { + return (*(int8_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int16_t *)minval == *(int16_t *)maxval) { + return (*(int16_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int32_t *)minval == *(int32_t *)maxval) { + return (*(int32_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(int64_t *)minval == *(int64_t *)maxval) { + return (*(int64_t *)minval != pFilter->filterInfo.lowerBndi); + } + + return true; +} + +bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(float *)minval == *(float *)maxval) { + return (*(float *)minval != pFilter->filterInfo.lowerBndd); + } + + return true; +} + +bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (*(double *)minval == *(double *)maxval) { + return (*(double *)minval != pFilter->filterInfo.lowerBndd); + } + + return true; +} + +bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if (pFilter->filterInfo.len > pFilter->bytes) { + return true; + } + + return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0; +} + +bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { + if (pFilter->filterInfo.len > pFilter->bytes) { + return true; + } + + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0; +} + +//////////////////////////////////////////////////////////////// + +bool rangeFilter_i32_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minvalfilterInfo.upperBndi &&*(int32_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval < pFilter->filterInfo.upperBndi && *(int32_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i32_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int32_t *)minval <= pFilter->filterInfo.upperBndi && *(int32_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +/////////////////////////////////////////////////////////////////////////////// +bool rangeFilter_i8_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minvalfilterInfo.upperBndi &&*(int8_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval < pFilter->filterInfo.upperBndi && *(int8_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i8_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int8_t *)minval <= pFilter->filterInfo.upperBndi && *(int8_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +///////////////////////////////////////////////////////////////////////////////////// +bool rangeFilter_i16_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minvalfilterInfo.upperBndi &&*(int16_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval < pFilter->filterInfo.upperBndi && *(int16_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i16_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int16_t *)minval <= pFilter->filterInfo.upperBndi && *(int16_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +//////////////////////////////////////////////////////////////////////// +bool rangeFilter_i64_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minvalfilterInfo.upperBndi &&*(int64_t *)maxval> pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval < pFilter->filterInfo.upperBndi && *(int64_t *)maxval >= pFilter->filterInfo.lowerBndi); +} + +bool rangeFilter_i64_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(int64_t *)minval <= pFilter->filterInfo.upperBndi && *(int64_t *)maxval > pFilter->filterInfo.lowerBndi); +} + +//////////////////////////////////////////////////////////////////////// +bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minvalfilterInfo.upperBndd &&*(float *)maxval> pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval < pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval > pFilter->filterInfo.lowerBndd); +} + +////////////////////////////////////////////////////////////////////////// +bool rangeFilter_dd_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minvalfilterInfo.upperBndd &&*(double *)maxval> pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval < pFilter->filterInfo.upperBndd && *(double *)maxval >= pFilter->filterInfo.lowerBndd); +} + +bool rangeFilter_dd_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return (*(double *)minval <= pFilter->filterInfo.upperBndd && *(double *)maxval > pFilter->filterInfo.lowerBndd); +} + +//////////////////////////////////////////////////////////////////////////// +bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i8, + large_i8, + equal_i8, + lessEqual_i8, + largeEqual_i8, + nequal_i8, + NULL, +}; + +bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i16, + large_i16, + equal_i16, + lessEqual_i16, + largeEqual_i16, + nequal_i16, + NULL, +}; + +bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i32, + large_i32, + equal_i32, + lessEqual_i32, + largeEqual_i32, + nequal_i32, + NULL, +}; + +bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_i64, + large_i64, + equal_i64, + lessEqual_i64, + largeEqual_i64, + nequal_i64, + NULL, +}; + +bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_ds, + large_ds, + equal_ds, + lessEqual_ds, + largeEqual_ds, + nequal_ds, + NULL, +}; + +bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + less_dd, + large_dd, + equal_dd, + lessEqual_dd, + largeEqual_dd, + nequal_dd, + NULL, +}; + +bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = { + NULL, + NULL, + NULL, + equal_str, + NULL, + NULL, + nequal_str, + like_str, +}; + +bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = { + NULL, + NULL, + NULL, + equal_nchar, + NULL, + NULL, + nequal_nchar, + like_nchar, +}; + +bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i8_ee, + rangeFilter_i8_ie, + rangeFilter_i8_ei, + rangeFilter_i8_ii, +}; + +bool (*rangeFilterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i16_ee, + rangeFilter_i16_ie, + rangeFilter_i16_ei, + rangeFilter_i16_ii, +}; + +bool (*rangeFilterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i32_ee, + rangeFilter_i32_ie, + rangeFilter_i32_ei, + rangeFilter_i32_ii, +}; + +bool (*rangeFilterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_i64_ee, + rangeFilter_i64_ie, + rangeFilter_i64_ei, + rangeFilter_i64_ii, +}; + +bool (*rangeFilterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_ds_ee, + rangeFilter_ds_ie, + rangeFilter_ds_ei, + rangeFilter_ds_ii, +}; + +bool (*rangeFilterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { + NULL, + rangeFilter_dd_ee, + rangeFilter_dd_ie, + rangeFilter_dd_ei, + rangeFilter_dd_ii, +}; + +__filter_func_t* getRangeFilterFuncArray(int32_t type) { + switch(type) { + case TSDB_DATA_TYPE_BOOL: return rangeFilterFunc_i8; + case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8; + case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16; + case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32; + case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter + case TSDB_DATA_TYPE_BIGINT: return rangeFilterFunc_i64; + case TSDB_DATA_TYPE_FLOAT: return rangeFilterFunc_ds; + case TSDB_DATA_TYPE_DOUBLE: return rangeFilterFunc_dd; + default:return NULL; + } +} + +__filter_func_t* getValueFilterFuncArray(int32_t type) { + switch(type) { + case TSDB_DATA_TYPE_BOOL: return filterFunc_i8; + case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8; + case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16; + case TSDB_DATA_TYPE_INT: return filterFunc_i32; + case TSDB_DATA_TYPE_TIMESTAMP: //timestamp uses bigint filter + case TSDB_DATA_TYPE_BIGINT: return filterFunc_i64; + case TSDB_DATA_TYPE_FLOAT: return filterFunc_ds; + case TSDB_DATA_TYPE_DOUBLE: return filterFunc_dd; + case TSDB_DATA_TYPE_BINARY: return filterFunc_str; + case TSDB_DATA_TYPE_NCHAR: return filterFunc_nchar; + default: return NULL; + } +} + +bool supportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 4ab6895863..2f644175a5 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -189,8 +189,8 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable for (int32_t i = 0; i < pCond->numOfCols; ++i) { SColumnInfoData pDest = {{0}, 0}; - pDest.info = pCond->colList[i].info; - pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCond->colList[i].info.bytes); + pDest.info = pCond->colList[i]; + pDest.pData = calloc(1, EXTRA_BYTES + bufferCapacity * pCond->colList[i].bytes); taosArrayPush(pQueryHandle->pColumns, &pDest); } @@ -595,10 +595,8 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, j); if (pCol->info.colId == colId) { - // SDataCol* pDataCol = &pCols->cols[i]; -// pCol->pData = pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start; - memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start, - pQueryHandle->realNumOfRows * pCol->info.bytes); + memmove(pCol->pData, pQueryHandle->rhelper.pDataCols[0]->cols[i].pData + pCol->info.bytes * start, + pQueryHandle->realNumOfRows * pCol->info.bytes); break; } } @@ -1082,7 +1080,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { return pHandle->pColumns; } else { STableBlockInfo* pBlockInfoEx = &pHandle->pDataBlockInfo[pHandle->cur.slot]; - STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo; + STableCheckInfo* pCheckInfo = pBlockInfoEx->pTableCheckInfo; SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfoEx->pBlock.compBlock); assert(pHandle->realNumOfRows <= binfo.rows); diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 28ebb18235..6fab24d51a 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -61,7 +61,7 @@ void taosArrayPop(SArray* pArray); * @param index * @return */ -void* taosArrayGet(SArray* pArray, size_t index); +void* taosArrayGet(const SArray* pArray, size_t index); /** * get the pointer data from the array @@ -69,7 +69,7 @@ void* taosArrayGet(SArray* pArray, size_t index); * @param index * @return */ -void* taosArrayGetP(SArray* pArray, size_t index); +void* taosArrayGetP(const SArray* pArray, size_t index); /** * return the size of array diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index d97b220a40..c025958438 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -84,12 +84,12 @@ void taosArrayPop(SArray* pArray) { pArray->size -= 1; } -void* taosArrayGet(SArray* pArray, size_t index) { +void* taosArrayGet(const SArray* pArray, size_t index) { assert(index < pArray->size); return TARRAY_GET_ELEM(pArray, index); } -void* taosArrayGetP(SArray* pArray, size_t index) { +void* taosArrayGetP(const SArray* pArray, size_t index) { void* ret = taosArrayGet(pArray, index); if (ret == NULL) { return NULL; From 47c2796bf753849e778140201a68d8b077ce389d Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 23 Apr 2020 12:22:39 +0800 Subject: [PATCH 04/63] [td-171] refactor client code --- src/client/inc/tscUtil.h | 49 ++- src/client/inc/tsclient.h | 38 +- src/client/src/tscAsync.c | 17 +- src/client/src/tscFunctionImpl.c | 8 +- src/client/src/tscLocal.c | 97 +++-- src/client/src/tscParseInsert.c | 6 +- src/client/src/tscSQLParser.c | 369 +++++++++--------- src/client/src/tscSecondaryMerge.c | 104 +++--- src/client/src/tscServer.c | 88 ++--- src/client/src/tscSql.c | 75 ++-- src/client/src/tscStream.c | 8 +- src/client/src/tscSubquery.c | 45 +-- src/client/src/tscUtil.c | 576 +++++++++-------------------- src/inc/taosmsg.h | 24 +- 14 files changed, 665 insertions(+), 839 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 70c2ed2c7a..723e5bf9bc 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -58,7 +58,7 @@ typedef struct SJoinSubquerySupporter { SLimitVal limit; // limit info uint64_t uid; // query meter uid SArray* colList; // previous query information - SSqlExprInfo exprsInfo; + SArray* exprsInfo; SFieldInfo fieldsInfo; STagCond tagCond; SSqlGroupbyExpr groupbyExpr; @@ -85,8 +85,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx); -STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); +UNUSED_FUNC STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); /** * @@ -115,49 +114,49 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex); -int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertOrImportData(char* sqlstr); /* use for keep current db info temporarily, for handle table with db prefix */ +// todo remove it void tscGetDBInfoFromMeterId(char* tableId, char* db); int tscAllocPayload(SSqlCmd* pCmd, int size); -void tscFieldInfoSetValFromSchema(SFieldInfo* pFieldInfo, int32_t index, SSchema* pSchema); -void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* pField); -void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, const char* name, int16_t bytes); -void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible); -void tscFieldInfoSetExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlExpr* pExpr); -void tscFieldInfoSetBinExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlFunctionExpr* pExpr); +TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes); + +SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField); +SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field); + +SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index); +TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo); void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size); void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src); void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo); -TAOS_FIELD* tscFieldInfoGetField(SQueryInfo* pQueryInfo, int32_t index); -int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); -int32_t tscGetResRowLength(SQueryInfo* pQueryInfo); -void tscClearFieldInfo(SFieldInfo* pFieldInfo); +int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); +void tscFieldInfoClear(SFieldInfo* pFieldInfo); int32_t tscNumOfFields(SQueryInfo* pQueryInfo); -int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2); +int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2); void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex); -SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize); -SSqlExpr* tscSqlExprInsertEmpty(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId); +int32_t tscGetResRowLength(SArray* pExprList); +SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize); +SSqlExpr* tscSqlExprAppend(SArray* exprInfo, int16_t functionId); SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size); int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo); SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index); -void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t uid, bool deepcopy); -void* tscSqlExprDestroy(SSqlExpr* pExpr); -void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo); +SArray* tscSqlExprCopy(const SArray* src, uint64_t uid, bool deepcopy); +void tscSqlExprInfoDestroy(SArray* pExprInfo); SColumn* tscColumnClone(const SColumn* src); SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); @@ -184,17 +183,16 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo); void tscSetFreeHeatBeat(STscObj* pObj); bool tscShouldFreeHeatBeat(SSqlObj* pHb); void tscCleanSqlCmd(SSqlCmd* pCmd); -bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql); +bool tscShouldBeFreed(SSqlObj* pSql); -void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); +void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex); STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); -STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index); -void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); +void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, int16_t numOfTags, int16_t* tags); @@ -204,7 +202,6 @@ int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); void tscFreeSubqueryInfo(SSqlCmd* pCmd); void tscClearSubqueryInfo(SSqlCmd* pCmd); -void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* keyStr, uint64_t uid); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index e1ff8ff20d..59cba3105f 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -34,8 +34,6 @@ extern "C" { #include "tsqlfunction.h" #include "queryExecutor.h" -#define TSC_GET_RESPTR_BASE(res, _queryinfo, col) (res->data + ((_queryinfo)->fieldsInfo.pSqlExpr[col]->offset) * res->numOfRows) - // forward declaration struct SSqlInfo; @@ -92,29 +90,18 @@ typedef struct SColumnIndex { int16_t columnIndex; } SColumnIndex; +typedef struct SFieldSupInfo { + bool visible; + SArithExprInfo* pArithExprInfo; + SSqlExpr* pSqlExpr; +} SFieldSupInfo; + typedef struct SFieldInfo { - int16_t numOfOutputCols; // number of column in result - int16_t numOfAlloc; // allocated size - TAOS_FIELD *pFields; - - /* - * define if this column is belong to the queried result, it may be add by parser to faciliate - * the query process - * - * NOTE: these hidden columns always locate at the end of the output columns - */ - bool * pVisibleCols; - int32_t numOfHiddenCols; // the number of column not belongs to the queried result columns - SSqlFunctionExpr** pExpr; // used for aggregation arithmetic express,such as count(*)+count(*) - SSqlExpr** pSqlExpr; + int16_t numOfOutput; // number of column in result + SArray* pFields; // SArray + SArray* pSupportInfo; // SArray } SFieldInfo; -typedef struct SSqlExprInfo { - int16_t numOfAlloc; - int16_t numOfExprs; - SSqlExpr** pExprs; -} SSqlExprInfo; - typedef struct SColumn { SColumnIndex colIndex; int32_t numOfFilters; @@ -193,7 +180,7 @@ typedef struct STableDataBlocks { SParamInfo *params; } STableDataBlocks; -typedef struct SDataBlockList { +typedef struct SDataBlockList { // todo remove uint32_t nSize; uint32_t nAlloc; STableDataBlocks **pData; @@ -209,9 +196,9 @@ typedef struct SQueryInfo { int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info - SArray* colList; + SArray* colList; // SArray SFieldInfo fieldsInfo; - SSqlExprInfo exprsInfo; + SArray* exprsInfo; // SArray SLimitVal limit; SLimitVal slimit; STagCond tagCond; @@ -441,6 +428,7 @@ int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *s void tscQueueAsyncFreeResult(SSqlObj *pSql); int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo); +char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column); extern void * pVnodeConn; extern void * pTscMgmtConn; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index ee38fec475..7357da5d17 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -287,9 +287,9 @@ void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows) { } for (int i = 0; i < pCmd->numOfCols; ++i){ - SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i]; - if (pExpr != NULL) { - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row; + SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + if (pSup->pSqlExpr != NULL) { +// pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pSup->pSqlExpr->resBytes * pRes->row; } else { //todo add } @@ -308,11 +308,12 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); for (int i = 0; i < pCmd->numOfCols; ++i) { - SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i]; - if (pExpr != NULL) { - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row; + SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + + if (pSup->pSqlExpr != NULL) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i); } else { - //todo add +// todo add } } @@ -332,7 +333,7 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) { int code = pRes->code; // in case of async insert, restore the user specified callback function - bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); + bool shouldFree = tscShouldBeFreed(pSql); if (cmd == TSDB_SQL_INSERT) { assert(pSql->fp != NULL); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index d3bf1c5206..92bd535d41 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3297,7 +3297,7 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { char *arithmetic_callback_function(void *param, char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - SSqlFunctionExpr *pExpr = pSupport->pExpr; + SArithExprInfo *pExpr = pSupport->pArithExpr; int32_t colIndex = -1; for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { @@ -3315,7 +3315,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, + tSQLBinaryExprCalcTraverse(sas->pArithExpr->binExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; @@ -3327,10 +3327,10 @@ static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; sas->offset = index; - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, + tSQLBinaryExprCalcTraverse(sas->pArithExpr->binExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, arithmetic_callback_function); - pCtx->aOutputBuf += pCtx->outputBytes/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; + pCtx->aOutputBuf += pCtx->outputBytes; } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 7987c529bc..6482b00654 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -130,13 +130,13 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { SSchema *pSchema = tscGetTableSchema(pMeta); for (int32_t i = 0; i < numOfRows; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i, pSchema[i].name, TSDB_COL_NAME_LEN); char *type = tDataTypeDesc[pSchema[i].type].aName; - pField = tscFieldInfoGetField(pQueryInfo, 1); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i, type, pField->bytes); int32_t bytes = pSchema[i].bytes; @@ -144,10 +144,10 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { bytes = bytes / TSDB_NCHAR_SIZE; } - pField = tscFieldInfoGetField(pQueryInfo, 2); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; - pField = tscFieldInfoGetField(pQueryInfo, 3); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); if (i >= tscGetNumOfColumns(pMeta) && tscGetNumOfTags(pMeta) != 0) { strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i, "tag", strlen("tag") + 1); @@ -162,18 +162,18 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { char *pTagValue = tsGetTagsValue(pMeta); for (int32_t i = numOfRows; i < totalNumOfRows; ++i) { // field name - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i, pSchema[i].name, TSDB_COL_NAME_LEN); // type name - pField = tscFieldInfoGetField(pQueryInfo, 1); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); char *type = tDataTypeDesc[pSchema[i].type].aName; strncpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i, type, pField->bytes); // type length int32_t bytes = pSchema[i].bytes; - pField = tscFieldInfoGetField(pQueryInfo, 2); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes / TSDB_NCHAR_SIZE; } @@ -181,7 +181,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; // tag value - pField = tscFieldInfoGetField(pQueryInfo, 3); + pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 3); char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; if (isNull(pTagValue, pSchema[i].type)) { @@ -236,31 +236,47 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, int32_t typeColLength, int32_t noteColLength) { int32_t rowLen = 0; - SSqlCmd *pCmd = &pSql->cmd; - pCmd->numOfCols = numOfCols; + SColumnIndex index = {0}; + + pSql->cmd.numOfCols = numOfCols; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); pQueryInfo->order.order = TSDB_ORDER_ASC; - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_BINARY, "Field", TSDB_COL_NAME_LEN); + TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_COL_NAME_LEN}; + strncpy(f.name, "Field", TSDB_COL_NAME_LEN); + + SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN); + rowLen += TSDB_COL_NAME_LEN; - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 1, TSDB_DATA_TYPE_BINARY, "Type", typeColLength); + f.bytes = typeColLength; + f.type = TSDB_DATA_TYPE_BINARY; + strncpy(f.name, "Type", TSDB_COL_NAME_LEN); + + pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, typeColLength); + rowLen += typeColLength; - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 2, TSDB_DATA_TYPE_INT, "Length", sizeof(int32_t)); + f.bytes = sizeof(int32_t); + f.type = TSDB_DATA_TYPE_INT; + strncpy(f.name, "Length", TSDB_COL_NAME_LEN); + + pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), sizeof(int32_t)); + rowLen += sizeof(int32_t); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 3, TSDB_DATA_TYPE_BINARY, "Note", noteColLength); - rowLen += noteColLength; + f.bytes = noteColLength; + f.type = TSDB_DATA_TYPE_BINARY; + strncpy(f.name, "Note", TSDB_COL_NAME_LEN); - //set the sqlexpr part - SColumnIndex index = {0}; - pQueryInfo->fieldsInfo.pSqlExpr[0] = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN); - pQueryInfo->fieldsInfo.pSqlExpr[1] = tscSqlExprInsert(pQueryInfo, 1, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, typeColLength); - pQueryInfo->fieldsInfo.pSqlExpr[2] = tscSqlExprInsert(pQueryInfo, 2, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), sizeof(int32_t)); - pQueryInfo->fieldsInfo.pSqlExpr[3] = tscSqlExprInsert(pQueryInfo, 3, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, noteColLength); - + pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, noteColLength); + + rowLen += noteColLength; return rowLen; } @@ -310,7 +326,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { } int32_t totalNumOfResults = pMetricMeta->numOfTables; - int32_t rowLen = tscGetResRowLength(pQueryInfo); + int32_t rowLen = tscGetResRowLength(pQueryInfo->exprsInfo); tscInitResObjForLocalQuery(pSql, totalNumOfResults, rowLen); @@ -321,7 +337,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { for (int32_t j = 0; j < pSidList->numOfSids; ++j) { STableIdInfo *pSidExt = tscGetMeterSidInfo(pSidList, j); - for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutput; ++k) { SColIndex *pColIndex = &tscSqlExprGet(pQueryInfo, k)->colInfo; int16_t offsetId = pColIndex->colIdx; @@ -329,7 +345,7 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) { assert(0); char * val = NULL;//pSidExt->tags + vOffset[offsetId]; - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, k); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, k); memcpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, k) * totalNumOfResults + pField->bytes * rowIdx, val, (size_t)pField->bytes); @@ -350,17 +366,17 @@ static int tscBuildMetricTagSqlFunctionResult(SSqlObj *pSql) { #if 0 SSuperTableMeta *pMetricMeta = tscGetMetaInfo(pQueryInfo, 0)->pMetricMeta; int32_t totalNumOfResults = 1; // count function only produce one result - int32_t rowLen = tscGetResRowLength(pQueryInfo); + int32_t rowLen = tscGetResRowLength(pQueryInfo->exprsInfo); tscInitResObjForLocalQuery(pSql, totalNumOfResults, rowLen); int32_t rowIdx = 0; for (int32_t i = 0; i < totalNumOfResults; ++i) { - for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutput; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->colInfo.colIdx == -1 && pExpr->functionId == TSDB_FUNC_COUNT) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, k); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, k); memcpy(pRes->data + tscFieldInfoGetOffset(pQueryInfo, i) * totalNumOfResults + pField->bytes * rowIdx, &pMetricMeta->numOfTables, sizeof(pMetricMeta->numOfTables)); @@ -388,7 +404,7 @@ static int tscProcessQueryTags(SSqlObj *pSql) { return pSql->res.code; } - SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr *pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); if (pExpr->functionId == TSDB_FUNC_COUNT) { return tscBuildMetricTagSqlFunctionResult(pSql); } else { @@ -399,7 +415,7 @@ static int tscProcessQueryTags(SSqlObj *pSql) { static void tscProcessCurrentUser(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, pSql->pTscObj->user, pExpr->aliasName, TSDB_USER_LEN); } @@ -414,7 +430,7 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, db, pExpr->aliasName, TSDB_DB_NAME_LEN); } @@ -422,14 +438,14 @@ static void tscProcessServerVer(SSqlObj *pSql) { const char* v = pSql->pTscObj->sversion; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, v, pExpr->aliasName, tListLen(pSql->pTscObj->sversion)); } static void tscProcessClientVer(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, version, pExpr->aliasName, strlen(version)); } @@ -449,7 +465,7 @@ static void tscProcessServStatus(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); tscSetLocalQueryResult(pSql, "1", pExpr->aliasName, 2); } @@ -462,13 +478,16 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); pQueryInfo->order.order = TSDB_ORDER_ASC; - tscClearFieldInfo(&pQueryInfo->fieldsInfo); + tscFieldInfoClear(&pQueryInfo->fieldsInfo); + + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_BINARY, columnName, valueLength); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_BINARY, columnName, valueLength); tscInitResObjForLocalQuery(pSql, 1, valueLength); - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); - pQueryInfo->fieldsInfo.pSqlExpr[0] = pQueryInfo->exprsInfo.pExprs[0]; + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, 0); + pInfo->pSqlExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); strncpy(pRes->data, val, pField->bytes); } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 2c203e4307..254ad1cb9f 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -779,7 +779,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - setMeterID(pSTableMeterMetaInfo, &sToken, pSql); + tscSetTableId(pSTableMeterMetaInfo, &sToken, pSql); strncpy(pTag->name, pSTableMeterMetaInfo->name, TSDB_TABLE_ID_LEN); code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); @@ -922,7 +922,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } - int32_t ret = setMeterID(pTableMetaInfo, &tableToken, pSql); + int32_t ret = tscSetTableId(pTableMetaInfo, &tableToken, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -1059,7 +1059,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) { goto _error_clean; } - if ((code = setMeterID(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { goto _error_clean; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index cc9721b1fd..c7be16c457 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -116,7 +116,7 @@ static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSql static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); -static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); +static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); /* * Used during parsing query sql. Since the query sql usually small in length, error position @@ -126,23 +126,6 @@ static int32_t invalidSqlErrMsg(char* dstBuffer, const char* errMsg) { return tscInvalidSQLErrMsg(dstBuffer, errMsg, NULL); } -static int32_t tscQueryOnlyMetricTags(SQueryInfo* pQueryInfo, bool* queryOnMetricTags) { - assert(QUERY_IS_STABLE_QUERY(pQueryInfo->type)); - - *queryOnMetricTags = true; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - - if (pExpr->functionId != TSDB_FUNC_TAGPRJ && - !(pExpr->functionId == TSDB_FUNC_COUNT && pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX)) { - *queryOnMetricTags = false; - break; - } - } - - return TSDB_CODE_SUCCESS; -} - static int setColumnFilterInfoForTimestamp(SQueryInfo* pQueryInfo, tVariant* pVar) { int64_t time = 0; const char* msg = "invalid timestamp"; @@ -243,7 +226,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { assert(pInfo->pDCLInfo->nTokens == 1); - if (setMeterID(pTableMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pzName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { @@ -376,7 +359,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (setMeterID(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -577,7 +560,9 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { * are available. */ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { @@ -631,7 +616,8 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { * check invalid SQL: * select count(tbname)/count(tag1)/count(tag2) from super_table_name interval(1d); */ - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -663,7 +649,7 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { } SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); SColumnList ids = getColumnList(1, 0, PRIMARYKEY_TIMESTAMP_COL_INDEX); @@ -708,7 +694,7 @@ int32_t parseSlidingClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { return TSDB_CODE_SUCCESS; } -int32_t setMeterID(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; @@ -1129,7 +1115,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } for (int32_t i = 0; i < pSelection->nExpr; ++i) { - int32_t outputIndex = pQueryInfo->exprsInfo.numOfExprs; + int32_t outputIndex = tscSqlExprNumOfExprs(pQueryInfo); tSQLExprItem* pItem = &pSelection->a[i]; // project on all fields @@ -1182,7 +1168,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputIndex, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), sizeof(double)); addExprParams(pExpr, arithmeticExprStr, TSDB_DATA_TYPE_BINARY, strlen(arithmeticExprStr), index.tableIndex); @@ -1198,22 +1184,23 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, "abc", NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); - if (pQueryInfo->fieldsInfo.pExpr[slot] == NULL) { - SSqlFunctionExpr* pFuncExpr = calloc(1, sizeof(SSqlFunctionExpr)); - tscFieldInfoSetBinExpr(&pQueryInfo->fieldsInfo, slot, pFuncExpr); + if (pInfo->pSqlExpr == NULL) { + SArithExprInfo* pFuncExpr = calloc(1, sizeof(SArithExprInfo)); + pInfo->pArithExprInfo = pFuncExpr; // arithmetic expression always return result in the format of double float - pFuncExpr->resBytes = sizeof(double); + pFuncExpr->bytes = sizeof(double); pFuncExpr->interResBytes = sizeof(double); - pFuncExpr->resType = TSDB_DATA_TYPE_DOUBLE; + pFuncExpr->type = TSDB_DATA_TYPE_DOUBLE; - SSqlBinaryExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; + SExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; tExprNode* pNode = NULL; // SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - - int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, &pQueryInfo->exprsInfo, pQueryInfo, NULL); + + int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprsInfo, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(pQueryInfo->msg, "invalid expression in select clause"); @@ -1225,15 +1212,17 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel for(int32_t k = 0; k < pBinExprInfo->numOfCols; ++k) { SColIndex* pCol = &pBinExprInfo->pReqColumns[k]; - for(int32_t f = 0; f < pQueryInfo->exprsInfo.numOfExprs; ++f) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for(int32_t f = 0; f < size; ++f) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, f); if (strcmp(pExpr->aliasName, pCol->name) == 0) { pCol->colIndex = f; break; } } - - assert(pCol->colIndex >= 0 && pCol->colIndex < pQueryInfo->exprsInfo.numOfExprs); + + assert(pCol->colIndex >= 0 && pCol->colIndex < size); tfree(pNode); } } @@ -1246,7 +1235,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - if (pQueryInfo->fieldsInfo.numOfOutputCols > TSDB_MAX_COLUMNS) { + if (pQueryInfo->fieldsInfo.numOfOutput > TSDB_MAX_COLUMNS) { return TSDB_CODE_INVALID_SQL; } } @@ -1284,9 +1273,10 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi for (int32_t i = 0; i < pIdList->num; ++i) { tscColumnListInsert(pQueryInfo->colList, &(pIdList->ids[i])); } - - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, outputIndex, type, fieldName, bytes); - tscFieldInfoSetExpr(&pQueryInfo->fieldsInfo, outputIndex, pSqlExpr); + + TAOS_FIELD f = tscCreateField(type, fieldName, bytes); + SFieldSupInfo* pInfo =tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f); + pInfo->pSqlExpr = pSqlExpr; return TSDB_CODE_SUCCESS; } @@ -1299,19 +1289,10 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIndex); int16_t functionId = (int16_t)((colIndex >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); - - if (functionId == TSDB_FUNC_TAGPRJ) { -// addRequiredTagColumn(pQueryInfo, colIndex - numOfCols, tableIndex); - pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; - } else { - pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; - } + pQueryInfo->type = (functionId == TSDB_FUNC_TAGPRJ)? TSDB_QUERY_TYPE_STABLE_QUERY:TSDB_QUERY_TYPE_PROJECTION_QUERY; SColumnIndex index = {tableIndex, colIndex}; - SSqlExpr* pExpr = - tscSqlExprInsert(pQueryInfo, outputIndex, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); - - return pExpr; + return tscSqlExprInsert(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); } void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex) { @@ -1365,7 +1346,7 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, functionId, pIndex, pColSchema->type, pColSchema->bytes, pColSchema->bytes); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); @@ -1415,8 +1396,8 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum int32_t addProjectionExprAndResultField(SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for table query is not allowed"; - - int32_t startPos = pQueryInfo->exprsInfo.numOfExprs; + + int32_t startPos = tscSqlExprNumOfExprs(pQueryInfo); if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -1493,10 +1474,10 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[pColIndex->columnIndex].name); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, resColIdx, functionID, pColIndex, type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, functionID, pColIndex, type, bytes, bytes); strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName)); - // for all querie, the timestamp column meeds to be loaded + // for all queries, the timestamp column needs to be loaded SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; tscColumnListInsert(pQueryInfo->colList, &index); @@ -1550,7 +1531,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprInsert(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } else { // count the number of meters created according to the metric if (getColumnIndexByName(pToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1565,13 +1546,13 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr } int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprInsert(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprInsert(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); @@ -1651,7 +1632,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr if (optr == TK_DIFF) { colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); SColumnList ids = getColumnList(1, 0, 0); @@ -1663,7 +1644,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return invalidSqlErrMsg(pQueryInfo->msg, msg6); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionID, &index, resultType, resultSize, resultSize); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, functionID, &index, resultType, resultSize, resultSize); if (optr == TK_LEASTSQUARES) { /* set the leastsquares parameters */ @@ -1855,7 +1836,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return TSDB_CODE_INVALID_SQL; } - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprInsert(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); @@ -1872,7 +1853,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr // set the first column ts for top/bottom query SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); const int32_t TS_COLUMN_INDEX = 0; SColumnList ids = getColumnList(1, 0, TS_COLUMN_INDEX); @@ -1881,7 +1862,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr colIndex += 1; // the first column is ts - pExpr = tscSqlExprInsert(pQueryInfo, colIndex, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprInsert(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0); } @@ -2273,8 +2254,9 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { int16_t bytes = 0; int16_t type = 0; int16_t intermediateBytes = 0; - - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t k = 0; k < size; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); int16_t functionId = aAggs[pExpr->functionId].stableFuncId; @@ -2305,8 +2287,9 @@ void tscRestoreSQLFunctionForMetricQuery(SQueryInfo* pQueryInfo) { if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return; } - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); @@ -2340,7 +2323,8 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { const char* msg3 = "function not support for super table query"; // filter sql function not supported by metric query yet. - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_METRIC) == 0) { invalidSqlErrMsg(pQueryInfo->msg, msg3); @@ -2366,7 +2350,8 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { int32_t startIdx = 0; - int32_t functionID = tscSqlExprGet(pQueryInfo, startIdx)->functionId; + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); + int32_t functionID = pExpr->functionId; // ts function can be simultaneously used with any other functions. if (functionID == TSDB_FUNC_TS || functionID == TSDB_FUNC_TS_DUMMY) { @@ -2377,7 +2362,9 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { // diff function cannot be executed with other function // arithmetic function can be executed with other arithmetic functions - for (int32_t i = startIdx + 1; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = startIdx + 1; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int16_t functionId = pExpr->functionId; @@ -2419,7 +2406,8 @@ void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // update tags column index for expression - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue @@ -3080,7 +3068,8 @@ static int32_t validateSQLExpr(tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnL return TSDB_CODE_INVALID_SQL; } - int32_t outputIndex = pQueryInfo->exprsInfo.numOfExprs; + int32_t outputIndex = tscSqlExprNumOfExprs(pQueryInfo); + tSQLExprItem item = {.pNode = pExpr, .aliasName = NULL}; // sql function in selection clause, append sql function info in pSqlCmd structure sequentially @@ -3927,8 +3916,8 @@ int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t o int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) { const char rep[] = {'(', ')', '*', ',', '.', '/', '\\', '+', '-', '%', ' '}; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - char* fieldName = tscFieldInfoGetField(pQueryInfo, i)->name; + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; for (int32_t j = 0; j < TSDB_COL_NAME_LEN && fieldName[j] != 0; ++j) { for (int32_t k = 0; k < tListLen(rep); ++k) { if (fieldName[j] == rep[k]) { @@ -3942,10 +3931,10 @@ int32_t tsRewriteFieldNameIfNecessary(SQueryInfo* pQueryInfo) { } // the column name may be identical, here check again - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - char* fieldName = tscFieldInfoGetField(pQueryInfo, i)->name; - for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutputCols; ++j) { - if (strncasecmp(fieldName, tscFieldInfoGetField(pQueryInfo, j)->name, TSDB_COL_NAME_LEN) == 0) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + char* fieldName = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->name; + for (int32_t j = i + 1; j < pQueryInfo->fieldsInfo.numOfOutput; ++j) { + if (strncasecmp(fieldName, tscFieldInfoGetField(&pQueryInfo->fieldsInfo, j)->name, TSDB_COL_NAME_LEN) == 0) { const char* msg = "duplicated column name in new table"; return invalidSqlErrMsg(pQueryInfo->msg, msg); } @@ -3968,9 +3957,11 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } - + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + if (pQueryInfo->defaultVal == NULL) { - pQueryInfo->defaultVal = calloc(pQueryInfo->exprsInfo.numOfExprs, sizeof(int64_t)); + pQueryInfo->defaultVal = calloc(size, sizeof(int64_t)); if (pQueryInfo->defaultVal == NULL) { return TSDB_CODE_CLI_OUT_OF_MEMORY; } @@ -3980,8 +3971,8 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { pQueryInfo->interpoType = TSDB_INTERPO_NONE; } else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) { pQueryInfo->interpoType = TSDB_INTERPO_NULL; - for (int32_t i = START_INTERPO_COL_IDX; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - TAOS_FIELD* pFields = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) { + TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); setNull((char*)&pQueryInfo->defaultVal[i], pFields->type, pFields->bytes); } } else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) { @@ -4002,19 +3993,17 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { if (tscIsPointInterpQuery(pQueryInfo)) { startPos = 0; - if (numOfFillVal > pQueryInfo->exprsInfo.numOfExprs) { - numOfFillVal = pQueryInfo->exprsInfo.numOfExprs; + if (numOfFillVal > size) { + numOfFillVal = size; } } else { - numOfFillVal = (pFillToken->nExpr > pQueryInfo->exprsInfo.numOfExprs) - ? pQueryInfo->exprsInfo.numOfExprs - : pFillToken->nExpr; + numOfFillVal = (pFillToken->nExpr > size) ? size : pFillToken->nExpr; } int32_t j = 1; for (int32_t i = startPos; i < numOfFillVal; ++i, ++j) { - TAOS_FIELD* pFields = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) { setNull((char*)(&pQueryInfo->defaultVal[i]), pFields->type, pFields->bytes); @@ -4026,13 +4015,15 @@ int32_t parseFillClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { return invalidSqlErrMsg(pQueryInfo->msg, msg); } } - - if ((pFillToken->nExpr < pQueryInfo->exprsInfo.numOfExprs) || - ((pFillToken->nExpr - 1 < pQueryInfo->exprsInfo.numOfExprs) && (tscIsPointInterpQuery(pQueryInfo)))) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + if ((pFillToken->nExpr < size) || + ((pFillToken->nExpr - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) { tVariantListItem* lastItem = &pFillToken->a[pFillToken->nExpr - 1]; - for (int32_t i = numOfFillVal; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - TAOS_FIELD* pFields = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = numOfFillVal; i < size; ++i) { + TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); if (pFields->type == TSDB_DATA_TYPE_BINARY || pFields->type == TSDB_DATA_TYPE_NCHAR) { setNull((char*)(&pQueryInfo->defaultVal[i]), pFields->type, pFields->bytes); @@ -4231,7 +4222,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (setMeterID(pTableMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, &(pAlterSQL->name), pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -4263,8 +4254,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (!validateOneTags(pCmd, &pFieldList->p[0])) { return TSDB_CODE_INVALID_SQL; } - - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, 0, &pFieldList->p[0]); + + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { const char* msg1 = "no tags can be dropped"; const char* msg2 = "only support one tag"; @@ -4301,8 +4292,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[128] = {0}; strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name1, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { const char* msg1 = "tag name too long"; const char* msg2 = "invalid tag name"; @@ -4336,13 +4328,15 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - char name[128] = {0}; + char name[TSDB_COL_NAME_LEN + 1] = {0}; strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); memset(name, 0, tListLen(name)); strncpy(name, pVarList->a[1].pVar.pz, pVarList->a[1].pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 1, TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { const char* msg1 = "invalid tag value"; const char* msg2 = "update normal column not supported"; @@ -4377,9 +4371,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[128] = {0}; strncpy(name1, pTagName->pz, pTagName->nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name1, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); - + + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + } else if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN) { tFieldList* pFieldList = pAlterSQL->pAddColumns; if (pFieldList->nField > 1) { @@ -4390,8 +4385,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (!validateOneColumn(pCmd, &pFieldList->p[0])) { return TSDB_CODE_INVALID_SQL; } - - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, 0, &pFieldList->p[0]); + + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) { const char* msg1 = "no columns can be dropped"; const char* msg2 = "only support one column"; @@ -4418,10 +4413,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg3); } - char name1[128] = {0}; + char name1[TSDB_COL_NAME_LEN + 1] = {0}; strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 0, TSDB_DATA_TYPE_INT, name1, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } return TSDB_CODE_SUCCESS; @@ -4434,8 +4429,9 @@ int32_t validateSqlFunctionInStreamSql(SQueryInfo* pQueryInfo) { if (pQueryInfo->intervalTime != 0 && pQueryInfo->intervalTime < 10) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + for (int32_t i = 0; i < size; ++i) { int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; if (!IS_STREAM_QUERY_VALID(aAggs[functId].nStatus)) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -4450,13 +4446,15 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SQueryInfo* pQueryInfo) { const char* msg1 = "column projection is not compatible with interval"; // multi-output set/ todo refactor - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t k = 0; k < size; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); // projection query on primary timestamp, the selectivity function needs to be present. if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { bool hasSelectivity = false; - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { SSqlExpr* pEx = tscSqlExprGet(pQueryInfo, j); if ((aAggs[pEx->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { hasSelectivity = true; @@ -4645,11 +4643,11 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* if (UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { bool queryOnTags = false; - if (tscQueryOnlyMetricTags(pQueryInfo, &queryOnTags) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_INVALID_SQL; - } +// if (tscQueryOnlyMetricTags(pQueryInfo, &queryOnTags) != TSDB_CODE_SUCCESS) { +// return TSDB_CODE_INVALID_SQL; +// } - if (queryOnTags == true) { // local handle the metric tag query + if (queryOnTags == true) { // local handle the super table tag query pQueryInfo->command = TSDB_SQL_RETRIEVE_TAGS; } else { if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { @@ -4707,9 +4705,11 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* if (pQueryInfo->slimit.limit != -1 || pQueryInfo->slimit.offset != 0) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + // filter the query functions operating on "tbname" column that are not supported by normal columns. - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); @@ -4811,14 +4811,14 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex) { // the first column not timestamp column, add it SSqlExpr* pExpr = NULL; - if (pQueryInfo->exprsInfo.numOfExprs > 0) { + if (tscSqlExprNumOfExprs(pQueryInfo) > 0) { pExpr = tscSqlExprGet(pQueryInfo, 0); } if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) { SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprInsert(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); pExpr->colInfo.flag = TSDB_COL_NORMAL; // NOTE: tag column does not add to source column list @@ -4833,9 +4833,9 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau if (pParentQueryInfo->groupbyExpr.numOfGroupCols > 0) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex); - int32_t num = pQueryInfo->exprsInfo.numOfExprs; - - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, num - 1); + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, size - 1); if (pExpr->functionId != TSDB_FUNC_TAG) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); @@ -4846,14 +4846,13 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau int16_t type = pSchema[index.columnIndex].type; int16_t bytes = pSchema[index.columnIndex].bytes; char* name = pSchema[index.columnIndex].name; - - pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_TAG, &index, type, bytes, - bytes); + + pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); pExpr->colInfo.flag = TSDB_COL_TAG; // NOTE: tag column does not add to source column list SColumnList ids = {0}; - insertResultField(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, &ids, bytes, type, name, pExpr); + insertResultField(pQueryInfo, size, &ids, bytes, type, name, pExpr); int32_t relIndex = index.columnIndex; @@ -4879,9 +4878,9 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index); SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = index}; - - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_PRJ, &colIndex, - pSchema->type, pSchema->bytes, pSchema->bytes); + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_PRJ, &colIndex, pSchema->type, pSchema->bytes, pSchema->bytes); pExpr->colInfo.flag = TSDB_COL_NORMAL; doLimitOutputNormalColOfGroupby(pExpr); @@ -4891,14 +4890,17 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { list.num = 1; list.ids[0] = colIndex; - insertResultField(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs - 1, &list, pSchema->bytes, pSchema->type, - pSchema->name, pExpr); - tscFieldInfoUpdateVisible(&pQueryInfo->fieldsInfo, pQueryInfo->exprsInfo.numOfExprs - 1, false); + insertResultField(pQueryInfo, size - 1, &list, pSchema->bytes, pSchema->type, pSchema->name, pExpr); + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, size - 1); + pInfo->visible = false; } static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { int32_t tagLength = 0; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || pExpr->functionId == TSDB_FUNC_TAG) { pExpr->functionId = TSDB_FUNC_TAG_DUMMY; @@ -4912,7 +4914,7 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) { SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex]; @@ -4923,7 +4925,9 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) { } static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_PRJ) { bool qualifiedCol = false; @@ -4954,8 +4958,9 @@ static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { bool hasTagPrj = false; bool hasColumnPrj = false; - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_PRJ) { hasColumnPrj = true; @@ -4971,7 +4976,8 @@ static bool onlyTagPrjFunction(SQueryInfo* pQueryInfo) { static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { bool allInGroupby = true; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_TAGPRJ) { continue; @@ -4988,7 +4994,9 @@ static bool allTagPrjInGroupby(SQueryInfo* pQueryInfo) { } static void updateTagPrjFunction(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = taosArrayGetSize(pQueryInfo->exprsInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ) { pExpr->functionId = TSDB_FUNC_TAG; @@ -5010,8 +5018,9 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { int16_t numOfSelectivity = 0; int16_t numOfAggregation = 0; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprsInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, i); if (pExpr->functionId == TSDB_FUNC_TAGPRJ || (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { tagColExists = true; @@ -5019,8 +5028,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { } } - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, i); + + int16_t functionId = pExpr->functionId; if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_ARITHM) { continue; @@ -5051,7 +5062,8 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo) { * If more than one selectivity functions exist, all the selectivity functions must be last_row. * Otherwise, return with error code. */ - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < numOfExprs; ++i) { + int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId == TSDB_FUNC_TAGPRJ) { continue; @@ -5107,12 +5119,12 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { bytes = pSchema[colIndex].bytes; name = pSchema[colIndex].name; } - + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + if (TSDB_COL_IS_TAG(pColIndex->flag)) { SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; - - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs, TSDB_FUNC_TAG, &index, - type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN); @@ -5121,7 +5133,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { // NOTE: tag column does not add to source column list SColumnList ids = getColumnList(1, 0, pColIndex->colIndex); - insertResultField(pQueryInfo, pQueryInfo->exprsInfo.numOfExprs-1, &ids, bytes, type, name, pExpr); + insertResultField(pQueryInfo, size - 1, &ids, bytes, type, name, pExpr); } else { // if this query is "group by" normal column, interval is not allowed if (pQueryInfo->intervalTime > 0) { @@ -5129,7 +5141,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { } bool hasGroupColumn = false; - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, j); if (pExpr->colInfo.colId == pColIndex->colId) { break; @@ -5172,7 +5184,8 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { } // check all query functions in selection clause, multi-output functions are not allowed - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int32_t functId = pExpr->functionId; @@ -5259,7 +5272,8 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { } } - SSqlExpr* pExpr1 = tscSqlExprInsertEmpty(pQueryInfo, 0, TSDB_FUNC_TAG_DUMMY); + SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo->exprsInfo, TSDB_FUNC_TAG_DUMMY); + const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name; strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName)); @@ -5361,7 +5375,8 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, subClauseIndex); - if (pQueryInfo->exprsInfo.numOfExprs == 0) { + int32_t size = tscSqlExprNumOfExprs(pQueryInfo); + if (size == 0) { return; } @@ -5370,8 +5385,8 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { char str[1024] = {0}; int32_t offset = 0; - offset += sprintf(str, "num:%d [", pQueryInfo->exprsInfo.numOfExprs); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + offset += sprintf(str, "num:%d [", size); + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); char tmpBuf[1024] = {0}; @@ -5382,7 +5397,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { offset += sprintf(str + offset, "%s", tmpBuf); - if (i < pQueryInfo->exprsInfo.numOfExprs - 1) { + if (i < size - 1) { str[offset++] = ','; } } @@ -5413,7 +5428,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (setMeterID(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -5424,14 +5439,14 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p int32_t col = 0; for (; col < pFieldList->nField; ++col) { - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, col, &pFieldList->p[col]); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[col]); } pCmd->numOfCols = (int16_t)pFieldList->nField; if (pTagList != NULL) { // create metric[optional] for (int32_t i = 0; i < pTagList->nField; ++i) { - tscFieldInfoSetValFromField(&pQueryInfo->fieldsInfo, col++, &pTagList->p[i]); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pTagList->p[col]); } pCmd->count = pTagList->nField; @@ -5468,7 +5483,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (setMeterID(pStableMeterMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pStableMeterMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5510,7 +5525,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } STableMetaInfo* pTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - int32_t ret = setMeterID(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); + int32_t ret = tscSetTableId(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -5548,7 +5563,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } - if (setMeterID(pTableMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, &srcToken, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg2); } @@ -5579,7 +5594,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set the created table[stream] name - if (setMeterID(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pTableMetaInfo, pzTableName, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); } @@ -5591,7 +5606,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_INVALID_SQL; } - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; if (validateSqlFunctionInStreamSql(pQueryInfo) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; @@ -5616,7 +5631,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set the number of stream table columns - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; return TSDB_CODE_SUCCESS; } @@ -5688,7 +5703,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { STableMetaInfo* pMeterInfo1 = tscGetMetaInfo(pQueryInfo, i); SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; - if (setMeterID(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableId(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5815,7 +5830,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_SUCCESS; // Does not build query message here } -int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { +int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; @@ -5848,10 +5863,14 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SSqlExprInfo* strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n); // set the input column data byte and type. - for (int32_t i = 0; i < pExprInfo->numOfExprs; ++i) { - if (strcmp((*pExpr)->pSchema->name, pExprInfo->pExprs[i]->aliasName) == 0) { - (*pExpr)->pSchema->type = pExprInfo->pExprs[i]->resType; - (*pExpr)->pSchema->bytes = pExprInfo->pExprs[i]->resBytes; + size_t size = taosArrayGetSize(pExprInfo); + + for (int32_t i = 0; i < size; ++i) { + SSqlExpr* p1 = taosArrayGetP(pExprInfo, i); + + if (strcmp((*pExpr)->pSchema->name, p1->aliasName) == 0) { + (*pExpr)->pSchema->type = p1->resType; + (*pExpr)->pSchema->bytes = p1->resBytes; break; } } diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index af186c84af..6631add297 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -61,7 +61,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu * merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object. */ SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { SQLFunctionCtx *pCtx = &pReducer->pCtx[i]; SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, i); @@ -108,10 +110,10 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu int16_t n = 0; int16_t tagLen = 0; - SQLFunctionCtx **pTagCtx = calloc(pQueryInfo->fieldsInfo.numOfOutputCols, POINTER_BYTES); + SQLFunctionCtx **pTagCtx = calloc(pQueryInfo->fieldsInfo.numOfOutput, POINTER_BYTES); SQLFunctionCtx *pCtx = NULL; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExpr->resBytes; @@ -254,8 +256,9 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd // the input data format follows the old format, but output in a new format. // so, all the input must be parsed as old format - pReducer->pCtx = (SQLFunctionCtx *)calloc(pQueryInfo->exprsInfo.numOfExprs, sizeof(SQLFunctionCtx)); - + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + pReducer->pCtx = (SQLFunctionCtx *)calloc(size, sizeof(SQLFunctionCtx)); pReducer->rowSize = pMemBuffer[0]->nElemSize; tscRestoreSQLFunctionForMetricQuery(pQueryInfo); @@ -279,7 +282,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16; pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage)); - int32_t finalRowLength = tscGetResRowLength(pQueryInfo); + int32_t finalRowLength = tscGetResRowLength(pQueryInfo->exprsInfo); pReducer->resColModel = finalmodel; pReducer->resColModel->capacity = pReducer->nResultBufSize / finalRowLength; assert(finalRowLength <= pReducer->rowSize); @@ -301,7 +304,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd } pReducer->pTempBuffer->numOfElems = 0; - pReducer->pResInfo = calloc((size_t)pQueryInfo->exprsInfo.numOfExprs, sizeof(SResultInfo)); + pReducer->pResInfo = calloc(size, sizeof(SResultInfo)); tscCreateResPointerInfo(pRes, pQueryInfo); tscInitSqlContext(pCmd, pRes, pReducer, pDesc); @@ -332,7 +335,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, pReducer->rowSize); - int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; + int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { pInterpoInfo->pTags[0] = (char *)pInterpoInfo->pTags + POINTER_BYTES * pQueryInfo->groupbyExpr.numOfGroupCols; @@ -462,7 +465,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { taosDestoryInterpoInfo(&pLocalReducer->interpolationInfo); if (pLocalReducer->pCtx != NULL) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; tVariantDestroy(&pCtx->tag); @@ -480,7 +483,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { tfree(pLocalReducer->pResultBuf); if (pLocalReducer->pResInfo != NULL) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { tfree(pLocalReducer->pResInfo[i].interResultBuf); } @@ -532,7 +535,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } if (numOfGroupByCols > 0) { - int32_t startCols = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; + int32_t startCols = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; // tags value locate at the last columns for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { @@ -612,8 +615,10 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; return pRes->code; } - - pSchema = (SSchema *)calloc(1, sizeof(SSchema) * pQueryInfo->exprsInfo.numOfExprs); + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + pSchema = (SSchema *)calloc(1, sizeof(SSchema) * size); if (pSchema == NULL) { tscError("%p failed to allocate memory", pSql); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; @@ -621,7 +626,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } int32_t rlen = 0; - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < size; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); pSchema[i].bytes = pExpr->resBytes; @@ -634,8 +639,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr if (rlen != 0) { capacity = nBufferSizes / rlen; } - - pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); + + pModel = createColumnModel(pSchema, size, capacity); size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups; for (int32_t i = 0; i < numOfSubs; ++i) { @@ -649,8 +654,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } // final result depends on the fields number - memset(pSchema, 0, sizeof(SSchema) * pQueryInfo->exprsInfo.numOfExprs); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + memset(pSchema, 0, sizeof(SSchema) * size); + for (int32_t i = 0; i < size; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); @@ -683,8 +688,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pSchema[i].bytes = bytes; strcpy(pSchema[i].name, pModel->pFields[i].field.name); } - - *pFinalModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity); + + *pFinalModel = createColumnModel(pSchema, size, capacity); tfree(pSchema); return TSDB_CODE_SUCCESS; @@ -800,7 +805,7 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo // static void reversedCopyResultToDstBuf(SQueryInfo* pQueryInfo, SSqlRes *pRes, tFilePage *pFinalDataPage) { // // for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { -// TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); +// TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); // // int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); // char * src = pFinalDataPage->data + (pRes->numOfRows - 1) * pField->bytes + pRes->numOfRows * offset; @@ -817,8 +822,10 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, SLocalReducer *pLocalReducer) { assert(0); - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); assert(offset == getColumnModelOffset(pLocalReducer->resColModel, i)); @@ -894,7 +901,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, &pLocalReducer->interpolationInfo); } - int32_t rowSize = tscGetResRowLength(pQueryInfo); + int32_t rowSize = tscGetResRowLength(pQueryInfo->exprsInfo); memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * rowSize); pFinalDataPage->numOfElems = 0; @@ -907,16 +914,16 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo int64_t actualETime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; - tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutputCols); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalReducer->resColModel->capacity); } - char ** srcData = (char **)malloc((POINTER_BYTES + sizeof(int32_t)) * pQueryInfo->fieldsInfo.numOfOutputCols); - int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(void *)); + char ** srcData = (char **)malloc((POINTER_BYTES + sizeof(int32_t)) * pQueryInfo->fieldsInfo.numOfOutput); + int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutput * sizeof(void *)); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { srcData[i] = pLocalReducer->pBufForInterpo + tscFieldInfoGetOffset(pQueryInfo, i) * pInterpoInfo->numOfRawDataInRows; functions[i] = tscSqlExprGet(pQueryInfo, i)->functionId; @@ -943,8 +950,8 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo newRows -= pQueryInfo->limit.offset; if (pQueryInfo->limit.offset > 0) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); memmove(pResPages[i]->data, pResPages[i]->data + pField->bytes * pQueryInfo->limit.offset, newRows * pField->bytes); } @@ -992,8 +999,8 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo } if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); } @@ -1003,7 +1010,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo } pFinalDataPage->numOfElems = 0; - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { tfree(pResPages[i]); } tfree(pResPages); @@ -1030,8 +1037,9 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, bool needInit) { // the tag columns need to be set before all functions execution SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, j); SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j]; @@ -1051,7 +1059,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, } } - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + for (int32_t j = 0; j < size; ++j) { int32_t functionId = tscSqlExprGet(pQueryInfo, j)->functionId; if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { continue; @@ -1071,8 +1079,9 @@ static void handleUnprocessedRow(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, tF static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) { int64_t maxOutput = 0; - - for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { + + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t j = 0; j < size; ++j) { // SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[j]; // if (pExpr == NULL) { // assert(pQueryInfo->fieldsInfo.pExpr[j] != NULL); @@ -1107,7 +1116,9 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) */ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLocalReducer *pLocalReducer) { int32_t maxBufSize = 0; // find the max tags column length to prepare the buffer - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t k = 0; k < size; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); if (maxBufSize < pExpr->resBytes && pExpr->functionId == TSDB_FUNC_TAG) { maxBufSize = pExpr->resBytes; @@ -1117,7 +1128,7 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo assert(maxBufSize >= 0); char *buf = malloc((size_t)maxBufSize); - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + for (int32_t k = 0; k < size; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); if (pExpr->functionId != TSDB_FUNC_TAG) { continue; @@ -1139,7 +1150,9 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo } int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t k = 0; k < size; ++k) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); aAggs[pExpr->functionId].xFinalize(&pLocalReducer->pCtx[k]); } @@ -1242,7 +1255,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no #endif SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; - int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; + int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutput - pQueryInfo->groupbyExpr.numOfGroupCols; for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { int16_t offset = getColumnModelOffset(pModel, startIndex + i); @@ -1258,7 +1271,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no } void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // reset output buffer to the beginning - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity; } @@ -1386,8 +1399,9 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) { SLocalReducer *pLocalReducer = pRes->pLocalReducer; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + for (int32_t k = 0; k < size; ++k) { SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, k); SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 377e4ed34c..ec4d2a927f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -359,13 +359,13 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { /* * Whether to free sqlObj or not should be decided before call the user defined function, since this SqlObj - * may be freed in UDF, and reused by other threads before tscShouldFreeAsyncSqlObj called, in which case - * tscShouldFreeAsyncSqlObj checks an object which is actually allocated by other threads. + * may be freed in UDF, and reused by other threads before tscShouldBeFreed called, in which case + * tscShouldBeFreed checks an object which is actually allocated by other threads. * * If this block of memory is re-allocated for an insert thread, in which tscKeepConn[command] equals to 0, - * the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately. + * the tscShouldBeFreed will success and tscFreeSqlObj free it immediately. */ - bool shouldFree = tscShouldFreeAsyncSqlObj(pSql); + bool shouldFree = tscShouldBeFreed(pSql); (*pSql->fp)(pSql->param, taosres, rpcMsg->code); if (shouldFree) { @@ -573,8 +573,10 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); int32_t srcColListSize = taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo); - - int32_t exprSize = sizeof(SSqlFuncExprMsg) * pQueryInfo->exprsInfo.numOfExprs; + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + int32_t exprSize = sizeof(SSqlFuncMsg) * numOfExprs; + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // meter query without tags values @@ -583,20 +585,6 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { } int32_t size = 4096; - -#if 0 - SSuperTableMeta *pMetricMeta = pTableMetaInfo->pMetricMeta; - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vgroupIndex); - - int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(STableIdInfo)) * pVnodeSidList->numOfSids; - int32_t outputColumnSize = pQueryInfo->exprsInfo.numOfExprs * sizeof(SSqlFuncExprMsg); - - int32_t size = meterInfoSize + outputColumnSize + srcColListSize + exprSize + MIN_QUERY_MSG_PKT_SIZE; - if (pQueryInfo->tsBuf != NULL) { - size += pQueryInfo->tsBuf->fileSize; - } -#endif - return size; } @@ -674,17 +662,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSql->ipList.ip[i] = pVgroupInfo->ipAddr[i].ip; } -#if 0 - SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pTableMetaInfo->vgroupIndex); - uint32_t vnodeId = pVnodeSidList->vpeerDesc[pVnodeSidList->index].vnode; - - numOfTables = pVnodeSidList->numOfSids; - if (numOfTables <= 0) { - tscError("%p vid:%d,error numOfTables in query message:%d", pSql, vnodeId, numOfTables); - return -1; // error - } -#endif - tscTrace("%p query on super table, numOfVgroup:%d, vgroupIndex:%d", pSql, pTableMetaInfo->vgroupList->numOfVgroups, index); pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); @@ -712,9 +689,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->queryType = htons(pQueryInfo->type); - pQueryMsg->numOfOutputCols = htons(pQueryInfo->exprsInfo.numOfExprs); - - int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutputCols; + + size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); + pQueryMsg->numOfOutput = htons(numOfOutput); if (numOfOutput < 0) { tscError("%p illegal value of number of output columns in query msg: %d", pSql, numOfOutput); return -1; @@ -773,7 +750,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { bool hasArithmeticFunction = false; - SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; + SSqlFuncMsg *pSqlFuncExpr = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); @@ -793,7 +770,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); - pMsg += sizeof(SSqlFuncExprMsg); + pMsg += sizeof(SSqlFuncMsg); for (int32_t j = 0; j < pExpr->numOfParams; ++j) { pSqlFuncExpr->arg[j].argType = htons((uint16_t)pExpr->param[j].nType); @@ -809,7 +786,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; + pSqlFuncExpr = (SSqlFuncMsg *)pMsg; } int32_t len = 0; @@ -855,7 +832,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { *((int64_t *)pMsg) = htobe64(pQueryInfo->defaultVal[i]); pMsg += sizeof(pQueryInfo->defaultVal[0]); } @@ -1260,7 +1237,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSchema = (SSchema *)pCreateTableMsg->schema; for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pSchema->type = pField->type; strcpy(pSchema->name, pField->name); @@ -1279,7 +1256,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - tscClearFieldInfo(&pQueryInfo->fieldsInfo); + tscFieldInfoClear(&pQueryInfo->fieldsInfo); msgLen = pMsg - (char*)pCreateTableMsg; pCreateTableMsg->contLen = htonl(msgLen); @@ -1327,7 +1304,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSchema *pSchema = pAlterTableMsg->schema; for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pSchema->type = pField->type; strcpy(pSchema->name, pField->name); @@ -1386,9 +1363,9 @@ static int tscSetResultPointer(SQueryInfo *pQueryInfo, SSqlRes *pRes) { return pRes->code; } - for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - pRes->tsrow[i] = (pRes->data + offset * pRes->numOfRows); + pRes->tsrow[i] = ((char*) pRes->data + offset * pRes->numOfRows); } return 0; @@ -2198,21 +2175,30 @@ int tscProcessShowRsp(SSqlObj *pSql) { pTableMetaInfo->pTableMeta = (STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer); - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if (pQueryInfo->colList == NULL) { pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + SColumnIndex index = {0}; for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i) { index.columnIndex = i; tscColumnListInsert(pQueryInfo->colList, &index); - tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pTableSchema[i]); + TAOS_FIELD field = { + .bytes = pSchema->bytes, + .type = pSchema->type, + }; + + strncpy(field.name, pSchema->name, TSDB_COL_NAME_LEN); + tscFieldInfoAppend(pFieldInfo, &field); - pQueryInfo->fieldsInfo.pSqlExpr[i] = tscSqlExprInsert(pQueryInfo, i, TSDB_FUNC_TS_DUMMY, &index, + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); + pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes); } @@ -2327,7 +2313,7 @@ int tscProcessQueryRsp(SSqlObj *pSql) { return 0; } -int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { +int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -2344,9 +2330,9 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) { tscSetResultPointer(pQueryInfo, pRes); if (pSql->pSubscription != NULL) { - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; + int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, numOfCols - 1); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; @@ -2629,7 +2615,7 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; tscProcessMsgRsp[TSDB_SQL_SELECT] = tscProcessQueryRsp; - tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromVnode; + tscProcessMsgRsp[TSDB_SQL_FETCH] = tscProcessRetrieveRspFromNode; tscProcessMsgRsp[TSDB_SQL_DROP_DB] = tscProcessDropDbRsp; tscProcessMsgRsp[TSDB_SQL_DROP_TABLE] = tscProcessDropTableRsp; @@ -2640,7 +2626,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiMeterMetaRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromVnode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_RETRIEVE_TAGS] = tscProcessTagRetrieveRsp; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 6fbd5e7739..6855ea8295 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -326,16 +326,21 @@ int taos_num_fields(TAOS_RES *res) { SSqlObj *pSql = (SSqlObj *)res; if (pSql == NULL || pSql->signature != pSql) return 0; + int32_t num = 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); if (pQueryInfo == NULL) { - return 0; + return num; } - SFieldInfo *pFieldsInfo = &pQueryInfo->fieldsInfo; - if (pFieldsInfo) - return (pFieldsInfo->numOfOutputCols - pFieldsInfo->numOfHiddenCols); - else - return 0; + size_t numOfCols = tscNumOfFields(pQueryInfo); + for(int32_t i = 0; i < numOfCols; ++i) { + SFieldSupInfo* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + if (pInfo->visible) { + num++; + } + } + + return num; } int taos_field_count(TAOS *taos) { @@ -357,11 +362,16 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { if (pSql == NULL || pSql->signature != pSql) return 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - - if (pQueryInfo) - return pQueryInfo->fieldsInfo.pFields; - else + if (pQueryInfo == NULL) { return NULL; + } + + size_t numOfCols = tscNumOfFields(pQueryInfo); + if (numOfCols == 0) { + return NULL; + } + + return pQueryInfo->fieldsInfo.pFields->pData; } int taos_retrieve(TAOS_RES *res) { @@ -414,8 +424,8 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { if (pQueryInfo == NULL) return 0; - for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i); + for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i); } *rows = pRes->tsrow; @@ -448,7 +458,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF static char *getArithemicInputSrc(void *param, char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - SSqlFunctionExpr * pExpr = pSupport->pExpr; + SArithExprInfo * pExpr = pSupport->pArithExpr; int32_t index = -1; for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { @@ -476,7 +486,8 @@ static void **doSetResultRowData(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); //todo refactor move away - for(int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for(int32_t k = 0; k < numOfExprs; ++k) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k); if (k > 0) { @@ -487,9 +498,9 @@ static void **doSetResultRowData(SSqlObj *pSql) { int32_t num = 0; for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - if (pQueryInfo->fieldsInfo.pSqlExpr[i] != NULL) { - SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i]; - pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); + if (pInfo->pSqlExpr != NULL) { + pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i) + pInfo->pSqlExpr->resBytes * pRes->row; } else { assert(0); } @@ -499,37 +510,37 @@ static void **doSetResultRowData(SSqlObj *pSql) { continue; } - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); transferNcharData(pSql, i, pField); - // calculate the result from serveral other columns - if (pQueryInfo->fieldsInfo.pExpr != NULL && pQueryInfo->fieldsInfo.pExpr[i] != NULL) { + // calculate the result from several other columns + if (pInfo->pArithExprInfo != NULL) { SArithmeticSupport *sas = (SArithmeticSupport *)calloc(1, sizeof(SArithmeticSupport)); sas->offset = 0; - sas->pExpr = pQueryInfo->fieldsInfo.pExpr[i]; + sas->pArithExpr = pInfo->pArithExprInfo; - sas->numOfCols = sas->pExpr->binExprInfo.numOfCols; + sas->numOfCols = sas->pArithExpr->binExprInfo.numOfCols; if (pRes->buffer[i] == NULL) { - pRes->buffer[i] = malloc(tscFieldInfoGetField(pQueryInfo, i)->bytes); + pRes->buffer[i] = malloc(tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->bytes); } for(int32_t k = 0; k < sas->numOfCols; ++k) { - int32_t columnIndex = sas->pExpr->binExprInfo.pReqColumns[k].colIndex; + int32_t columnIndex = sas->pArithExpr->binExprInfo.pReqColumns[k].colIndex; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); sas->elemSize[k] = pExpr->resBytes; sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; } - tSQLBinaryExprCalcTraverse(sas->pExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); + tSQLBinaryExprCalcTraverse(sas->pArithExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); pRes->tsrow[i] = pRes->buffer[i]; free(sas); //todo optimization } } - assert(num <= pQueryInfo->fieldsInfo.numOfOutputCols); + assert(num <= pQueryInfo->fieldsInfo.numOfOutput); pRes->row++; // index increase one-step return pRes->tsrow; @@ -591,11 +602,13 @@ static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) { static UNUSED_FUNC void **tscBuildResFromSubqueries(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; - + while (1) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + if (pRes->tsrow == NULL) { - pRes->tsrow = calloc(pQueryInfo->exprsInfo.numOfExprs, POINTER_BYTES); + pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); } bool success = false; @@ -619,7 +632,9 @@ static UNUSED_FUNC void **tscBuildResFromSubqueries(SSqlObj *pSql) { } if (success) { // current row of final output has been built, return to app - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < numOfExprs; ++i) { int32_t tableIndex = pRes->pColumnIndex[i].tableIndex; int32_t columnIndex = pRes->pColumnIndex[i].columnIndex; @@ -1126,7 +1141,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t return code; } - if ((code = setMeterID(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableId(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { return code; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 9323a4b0c4..4802bb293a 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -36,7 +36,7 @@ static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t l } static bool isProjectStream(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->functionId != TSDB_FUNC_PRJ) { return false; @@ -219,9 +219,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf void *oldPtr = pSql->res.data; pSql->res.data = tmpRes; - for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 1; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); assignVal(pSql->res.data + offset, (char *)(&pQueryInfo->defaultVal[i]), pField->bytes, pField->type); row[i] = pSql->res.data + offset; @@ -231,7 +231,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf row[0] = pRes->data; // char result[512] = {0}; - // taos_print_row(result, row, pQueryInfo->fieldsInfo.pFields, pQueryInfo->fieldsInfo.numOfOutputCols); + // taos_print_row(result, row, pQueryInfo->fieldsInfo.pFields, pQueryInfo->fieldsInfo.numOfOutput); // tscPrint("%p stream:%p query result: %s", pSql, pStream, result); tscTrace("%p stream:%p fetch result", pSql, pStream); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 5b0bba4607..46e859c221 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -190,10 +190,10 @@ void tscDestroyJoinSupporter(SJoinSubquerySupporter* pSupporter) { return; } - tscSqlExprInfoDestroy(&pSupporter->exprsInfo); + tscSqlExprInfoDestroy(pSupporter->exprsInfo); tscColumnListDestroy(pSupporter->colList); - tscClearFieldInfo(&pSupporter->fieldsInfo); + tscFieldInfoClear(&pSupporter->fieldsInfo); if (pSupporter->f != NULL) { fclose(pSupporter->f); @@ -238,7 +238,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { for (int32_t i = 0; i < pSql->numOfSubs; ++i) { pSupporter = pSql->pSubs[i]->param; - if (pSupporter->exprsInfo.numOfExprs > 0) { + if (taosArrayGetSize(pSupporter->exprsInfo) > 0) { ++numOfSub; } } @@ -264,7 +264,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pSupporter = pPrevSub->param; - if (pSupporter->exprsInfo.numOfExprs == 0) { + if (taosArrayGetSize(pSupporter->exprsInfo) == 0) { tscTrace("%p subIndex: %d, not need to launch query, ignore it", pSql, i); tscDestroyJoinSupporter(pSupporter); @@ -279,7 +279,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pSubQueryInfo->tsBuf = NULL; // free result for async object will also free sqlObj - assert(pSubQueryInfo->exprsInfo.numOfExprs == 1); // ts_comp query only requires one resutl columns + assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one resutl columns taos_free_result(pPrevSub); SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); @@ -304,17 +304,17 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { tscColumnListAssign(pQueryInfo->colList, pSupporter->colList, 0); tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); - tscSqlExprCopy(&pQueryInfo->exprsInfo, &pSupporter->exprsInfo, pSupporter->uid, false); + pQueryInfo->exprsInfo = tscSqlExprCopy(pSupporter->exprsInfo, pSupporter->uid, false); tscFieldInfoCopyAll(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); - pSupporter->exprsInfo.numOfExprs = 0; - pSupporter->fieldsInfo.numOfOutputCols = 0; + pSupporter->fieldsInfo.numOfOutput = 0; /* * if the first column of the secondary query is not ts function, add this function. * Because this column is required to filter with timestamp after intersecting. */ - if (pSupporter->exprsInfo.pExprs[0]->functionId != TSDB_FUNC_TS) { + SSqlExpr* pExpr = taosArrayGet(pSupporter->exprsInfo, 0); + if (pExpr->functionId != TSDB_FUNC_TS) { tscAddTimestampColumn(pQueryInfo, TSDB_FUNC_TS, 0); } @@ -347,8 +347,8 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscTrace("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); + taosArrayGetSize(pNewQueryInfo->exprsInfo), numOfCols, + pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); } //prepare the subqueries object failed, abort @@ -691,9 +691,9 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutputCols); + pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutput); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int32_t tableIndexOfSub = -1; @@ -710,7 +710,8 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; SQueryInfo* pSubQueryInfo = tscGetQueryInfoDetail(pSubCmd, 0); - for (int32_t k = 0; k < pSubQueryInfo->exprsInfo.numOfExprs; ++k) { + size_t numOfExprs = taosArrayGetSize(pSubQueryInfo->exprsInfo); + for (int32_t k = 0; k < numOfExprs; ++k) { SSqlExpr* pSubExpr = tscSqlExprGet(pSubQueryInfo, k); if (pExpr->functionId == pSubExpr->functionId && pExpr->colInfo.colId == pSubExpr->colInfo.colId) { pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; @@ -726,11 +727,6 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { // int32_t idx = pSql->cmd.vnodeIdx; - // SVnodeSidList *vnodeInfo = NULL; - // if (pTableMetaInfo->pMetricMeta != NULL) { - // vnodeInfo = tscGetVnodeSidList(pTableMetaInfo->pMetricMeta, idx - 1); - // } - SJoinSubquerySupporter* pSupporter = (SJoinSubquerySupporter*)param; // if (atomic_add_fetch_32(pSupporter->numOfComplete, 1) >= @@ -861,8 +857,8 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu } tscColumnListAssign(pSupporter->colList, pNewQueryInfo->colList, 0); - - tscSqlExprCopy(&pSupporter->exprsInfo, &pNewQueryInfo->exprsInfo, pSupporter->uid, false); + + pSupporter->exprsInfo = tscSqlExprCopy(pNewQueryInfo->exprsInfo, pSupporter->uid, false); tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond); @@ -876,8 +872,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); // this data needs to be transfer to support struct - pNewQueryInfo->fieldsInfo.numOfOutputCols = 0; - pNewQueryInfo->exprsInfo.numOfExprs = 0; + pNewQueryInfo->fieldsInfo.numOfOutput = 0; // set the ts,tags that involved in join, as the output column of intermediate result tscClearSubqueryInfo(&pNew->cmd); @@ -913,8 +908,8 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu tscTrace("%p subquery:%p tableIndex:%d, vnodeIdx:%d, type:%d, transfer to ts_comp query to retrieve timestamps, " "exprInfo:%d, colList:%d, fieldsInfo:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, - pNewQueryInfo->exprsInfo.numOfExprs, numOfCols, - pNewQueryInfo->fieldsInfo.numOfOutputCols, pNewQueryInfo->pTableMetaInfo[0]->name); + tscSqlExprNumOfExprs(pNewQueryInfo), numOfCols, + pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); tscPrintSelectClause(pNew, 0); } else { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index cb5e8c0d66..7de2f0ce70 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -30,69 +30,6 @@ #include "ttokendef.h" #include "tscLog.h" -/* - * the detailed information regarding metric meta key is: - * fullmetername + '.' + tagQueryCond + '.' + tableNameCond + '.' + joinCond + - * '.' + relation + '.' + [tagId1, tagId2,...] + '.' + group_orderType - * - * if querycond/tablenameCond/joinCond is null, its format is: - * fullmetername + '.' + '(nil)' + '.' + '(nil)' + relation + '.' + [tagId1, - * tagId2,...] + '.' + group_orderType - */ -void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) { - int32_t index = -1; - STableMetaInfo* pTableMetaInfo = tscGetMeterMetaInfoByUid(pQueryInfo, uid, &index); - - int32_t len = 0; - char tagIdBuf[128] = {0}; - for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { - len += sprintf(&tagIdBuf[len], "%d,", pTableMetaInfo->tagColumnIndex[i]); - } - - STagCond* pTagCond = &pQueryInfo->tagCond; - assert(len < tListLen(tagIdBuf)); - - const int32_t maxKeySize = TSDB_MAX_TAGS_LEN; // allowed max key size - - SCond* cond = tsGetSTableQueryCond(pTagCond, uid); - - char join[512] = {0}; - if (pTagCond->joinInfo.hasJoin) { - sprintf(join, "%s,%s", pTagCond->joinInfo.left.tableId, pTagCond->joinInfo.right.tableId); - } - - // estimate the buffer size - size_t tbnameCondLen = pTagCond->tbnameCond.cond != NULL ? strlen(pTagCond->tbnameCond.cond) : 0; - size_t redundantLen = 20; - - size_t bufSize = strlen(pTableMetaInfo->name) + tbnameCondLen + strlen(join) + strlen(tagIdBuf); - if (cond != NULL && cond->cond != NULL) { - bufSize += strlen(cond->cond); - } - - bufSize = (size_t)((bufSize + redundantLen) * 1.5); - char* tmp = calloc(1, bufSize); - - int32_t keyLen = snprintf(tmp, bufSize, "%s,%s,%s,%d,%s,[%s],%d", pTableMetaInfo->name, - ((cond != NULL && cond->cond != NULL) ? cond->cond : NULL), (tbnameCondLen > 0 ? pTagCond->tbnameCond.cond : NULL), - pTagCond->relType, join, tagIdBuf, pQueryInfo->groupbyExpr.orderType); - - assert(keyLen <= bufSize); - - if (keyLen < maxKeySize) { - strcpy(str, tmp); - } else { // using md5 to hash - MD5_CTX ctx; - MD5Init(&ctx); - - MD5Update(&ctx, (uint8_t*)tmp, keyLen); - char* pStr = base64_encode(ctx.digest, tListLen(ctx.digest)); - strcpy(str, pStr); - } - - free(tmp); -} - SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { if (pTagCond->pCond == NULL) { return NULL; @@ -138,7 +75,7 @@ bool tscQueryOnSTable(SSqlCmd* pCmd) { } bool tscQueryTags(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { if (tscSqlExprGet(pQueryInfo, i)->functionId != TSDB_FUNC_TAGPRJ) { return false; } @@ -153,7 +90,7 @@ bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functId == TSDB_FUNC_TAG_DUMMY) { hasTags = true; @@ -186,29 +123,6 @@ void tscGetDBInfoFromMeterId(char* tableId, char* db) { db[0] = 0; } -SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx) { -#if 0 - if (pMetricmeta == NULL) { - tscError("illegal metricmeta"); - return 0; - } - - if (pMetricmeta->numOfVnodes == 0 || pMetricmeta->numOfTables == 0) { - return 0; - } - - if (vnodeIdx < 0 || vnodeIdx >= pMetricmeta->numOfVnodes) { - int32_t vnodeRange = (pMetricmeta->numOfVnodes > 0) ? (pMetricmeta->numOfVnodes - 1) : 0; - tscError("illegal vnodeIdx:%d, reset to 0, vnodeIdx range:%d-%d", vnodeIdx, 0, vnodeRange); - - vnodeIdx = 0; - } - - return (SVnodeSidList*)(pMetricmeta->list[vnodeIdx] + (char*)pMetricmeta); -#endif - return NULL; -} - STableIdInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) { if (pSidList == NULL) { tscError("illegal sidlist"); @@ -271,8 +185,9 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { * 1. failed to get metermeta from server; 2. not a super table; 3. limitation is 0; * 4. show queries, instead of a select query */ + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo) || - pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pQueryInfo->exprsInfo.numOfExprs == 0) { + pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) { return false; } @@ -282,7 +197,7 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // for project query, only the following two function is allowed - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < numOfExprs; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) { @@ -312,7 +227,7 @@ bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableInde } bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TS) { return false; @@ -323,7 +238,9 @@ bool tscProjectionQueryOnTable(SQueryInfo* pQueryInfo) { } bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < size; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr == NULL) { return false; @@ -342,7 +259,8 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) { } bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr == NULL) { continue; @@ -368,11 +286,11 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) { int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { if (pRes->tsrow == NULL) { - int32_t numOfOutputCols = pQueryInfo->fieldsInfo.numOfOutputCols; - pRes->numOfCols = numOfOutputCols; + int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; + pRes->numOfCols = numOfOutput; - pRes->tsrow = calloc(POINTER_BYTES, numOfOutputCols); - pRes->buffer = calloc(POINTER_BYTES, numOfOutputCols); + pRes->tsrow = calloc(POINTER_BYTES, numOfOutput); + pRes->buffer = calloc(POINTER_BYTES, numOfOutput); // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { @@ -883,139 +801,71 @@ int tscAllocPayload(SSqlCmd* pCmd, int size) { return TSDB_CODE_SUCCESS; } -static void ensureSpace(SFieldInfo* pFieldInfo, int32_t size) { - if (size > pFieldInfo->numOfAlloc) { - int32_t oldSize = pFieldInfo->numOfAlloc; +TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) { + TAOS_FIELD f = { .type = type, .bytes = bytes, }; + strncpy(f.name, name, TSDB_COL_NAME_LEN); + return f; +} - int32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1); - while (newSize < size) { - newSize = (newSize << 1); - } - - if (newSize > TSDB_MAX_COLUMNS) { - newSize = TSDB_MAX_COLUMNS; - } - - int32_t inc = newSize - oldSize; - - pFieldInfo->pFields = realloc(pFieldInfo->pFields, newSize * sizeof(TAOS_FIELD)); - memset(&pFieldInfo->pFields[oldSize], 0, inc * sizeof(TAOS_FIELD)); - -// pFieldInfo->pOffset = realloc(pFieldInfo->pOffset, newSize * sizeof(int16_t)); -// memset(&pFieldInfo->pOffset[oldSize], 0, inc * sizeof(int16_t)); - - pFieldInfo->pVisibleCols = realloc(pFieldInfo->pVisibleCols, newSize * sizeof(bool)); - memset(&pFieldInfo->pVisibleCols[oldSize], 0, inc * sizeof(bool)); - - pFieldInfo->pSqlExpr = realloc(pFieldInfo->pSqlExpr, POINTER_BYTES*newSize); - pFieldInfo->pExpr = realloc(pFieldInfo->pExpr, POINTER_BYTES*newSize); +SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { + assert(pFieldInfo != NULL); + taosArrayPush(pFieldInfo->pFields, pField); + pFieldInfo->numOfOutput++; - memset(&pFieldInfo->pSqlExpr[oldSize], 0, inc * POINTER_BYTES); - memset(&pFieldInfo->pExpr[oldSize], 0, inc * POINTER_BYTES); + struct SFieldSupInfo info = { + .pSqlExpr = NULL, + .pArithExprInfo = NULL, + .visible = true, + }; - pFieldInfo->numOfAlloc = newSize; - } + return taosArrayPush(pFieldInfo->pSupportInfo, &info); } -static void evic(SFieldInfo* pFieldInfo, int32_t index) { - if (index < pFieldInfo->numOfOutputCols) { - memmove(&pFieldInfo->pFields[index + 1], &pFieldInfo->pFields[index], - sizeof(pFieldInfo->pFields[0]) * (pFieldInfo->numOfOutputCols - index)); - - memmove(&pFieldInfo->pVisibleCols[index + 1], &pFieldInfo->pVisibleCols[index], - sizeof(pFieldInfo->pVisibleCols[0]) * (pFieldInfo->numOfOutputCols - index)); - - memmove(&pFieldInfo->pSqlExpr[index + 1], &pFieldInfo->pSqlExpr[index], - sizeof(pFieldInfo->pSqlExpr[0]) * (pFieldInfo->numOfOutputCols - index)); +SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index) { + return taosArrayGet(pFieldInfo->pSupportInfo, index); +} + +SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { + taosArrayInsert(pFieldInfo->pFields, index, field); + pFieldInfo->numOfOutput++; - memmove(&pFieldInfo->pExpr[index + 1], &pFieldInfo->pExpr[index], - sizeof(pFieldInfo->pExpr[0]) * (pFieldInfo->numOfOutputCols - index)); - } -} - -static void setValueImpl(TAOS_FIELD* pField, int8_t type, const char* name, int16_t bytes) { - pField->type = type; - strncpy(pField->name, name, TSDB_COL_NAME_LEN); - pField->bytes = bytes; -} - -void tscFieldInfoSetValFromSchema(SFieldInfo* pFieldInfo, int32_t index, SSchema* pSchema) { - ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); - evic(pFieldInfo, index); - - TAOS_FIELD* pField = &pFieldInfo->pFields[index]; - setValueImpl(pField, pSchema->type, pSchema->name, pSchema->bytes); - pFieldInfo->numOfOutputCols++; -} - -void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* pField) { - ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); - evic(pFieldInfo, index); - - memcpy(&pFieldInfo->pFields[index], pField, sizeof(TAOS_FIELD)); - pFieldInfo->pVisibleCols[index] = true; - pFieldInfo->numOfOutputCols++; -} - -void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible) { - if (index < 0 || index >= pFieldInfo->numOfOutputCols) { - return; - } - - bool oldVisible = pFieldInfo->pVisibleCols[index]; - pFieldInfo->pVisibleCols[index] = visible; - - if (oldVisible != visible) { - if (!visible) { - pFieldInfo->numOfHiddenCols += 1; - } else { - if (pFieldInfo->numOfHiddenCols > 0) { - pFieldInfo->numOfHiddenCols -= 1; - } - } - } -} - -void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, const char* name, int16_t bytes) { - ensureSpace(pFieldInfo, pFieldInfo->numOfOutputCols + 1); - evic(pFieldInfo, index); - - TAOS_FIELD* pField = &pFieldInfo->pFields[index]; - setValueImpl(pField, type, name, bytes); - - pFieldInfo->pVisibleCols[index] = true; - pFieldInfo->numOfOutputCols++; -} - -void tscFieldInfoSetExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlExpr* pExpr) { - assert(index >= 0 && index < pFieldInfo->numOfOutputCols); - pFieldInfo->pSqlExpr[index] = pExpr; -} - -void tscFieldInfoSetBinExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlFunctionExpr* pExpr) { - assert(index >= 0 && index < pFieldInfo->numOfOutputCols); - pFieldInfo->pExpr[index] = pExpr; + struct SFieldSupInfo info = { + .pSqlExpr = NULL, + .pArithExprInfo = NULL, + .visible = true, + }; + + return taosArrayInsert(pFieldInfo->pSupportInfo, index, &info); } void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo) { - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - pExprInfo->pExprs[0]->offset = 0; + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t i = 1; i < pExprInfo->numOfExprs; ++i) { - pExprInfo->pExprs[i]->offset = pExprInfo->pExprs[i - 1]->offset + pExprInfo->pExprs[i - 1]->resBytes; + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + pExpr->offset = 0; + + for (int32_t i = 1; i < numOfExprs; ++i) { + SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprsInfo, i - 1); + SSqlExpr* p = taosArrayGetP(pQueryInfo->exprsInfo, i); + + p->offset = prev->offset + prev->resBytes; } } void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) { - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - if (pExprInfo->numOfExprs == 0) { + if (tscSqlExprNumOfExprs(pQueryInfo) == 0) { return; } - pExprInfo->pExprs[0]->offset = 0; + SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); + pExpr->offset = 0; - for (int32_t i = 1; i < pExprInfo->numOfExprs; ++i) { - pExprInfo->pExprs[i]->offset = pExprInfo->pExprs[i - 1]->offset + pExprInfo->pExprs[i - 1]->resBytes; + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 1; i < numOfExprs; ++i) { + SSqlExpr* prev = taosArrayGetP(pQueryInfo->exprsInfo, i - 1); + SSqlExpr* p = taosArrayGetP(pQueryInfo->exprsInfo, i); + + p->offset = prev->offset + prev->resBytes; } } @@ -1025,62 +875,52 @@ void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList } if (size <= 0) { - *dst = *src; tscFieldInfoCopyAll(dst, src); } else { // only copy the required column for (int32_t i = 0; i < size; ++i) { - assert(indexList[i] >= 0 && indexList[i] <= src->numOfOutputCols); - tscFieldInfoSetValFromField(dst, i, &src->pFields[indexList[i]]); - dst->pVisibleCols[i] = src->pVisibleCols[indexList[i]]; - dst->pSqlExpr[i] = src->pSqlExpr[indexList[i]]; + assert(indexList[i] >= 0 && indexList[i] <= src->numOfOutput); + TAOS_FIELD* p = taosArrayGet(src->pFields, indexList[i]); + SFieldSupInfo* pInfo = taosArrayGet(src->pSupportInfo, indexList[i]); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(dst, p); + *pInfo1 = *pInfo; } } } void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src) { - *dst = *src; + dst->numOfOutput = src->numOfOutput; - dst->pFields = malloc(sizeof(TAOS_FIELD) * dst->numOfAlloc); - dst->pVisibleCols = malloc(sizeof(bool) * dst->numOfAlloc); - dst->pSqlExpr = malloc(POINTER_BYTES * dst->numOfAlloc); - dst->pExpr = malloc(POINTER_BYTES * dst->numOfAlloc); - - memcpy(dst->pFields, src->pFields, sizeof(TAOS_FIELD) * dst->numOfOutputCols); - memcpy(dst->pVisibleCols, src->pVisibleCols, sizeof(bool) * dst->numOfOutputCols); - memcpy(dst->pSqlExpr, src->pSqlExpr, POINTER_BYTES * dst->numOfOutputCols); - memcpy(dst->pExpr, src->pExpr, POINTER_BYTES * dst->numOfOutputCols); + taosArrayCopy(dst->pFields, src->pFields); + taosArrayCopy(dst->pSupportInfo, src->pSupportInfo); } -TAOS_FIELD* tscFieldInfoGetField(SQueryInfo* pQueryInfo, int32_t index) { - if (index >= pQueryInfo->fieldsInfo.numOfOutputCols) { - return NULL; - } - - return &pQueryInfo->fieldsInfo.pFields[index]; +TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { + return taosArrayGet(pFieldInfo->pFields, index); } -int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutputCols; } +int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; } int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { - if (index >= pQueryInfo->exprsInfo.numOfExprs) { - return 0; - } + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index); + assert(pInfo != NULL); - return pQueryInfo->exprsInfo.pExprs[index]->offset; + return pInfo->pSqlExpr->offset; } -int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2) { +int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2) { assert(pFieldInfo1 != NULL && pFieldInfo2 != NULL); - if (pFieldInfo1->numOfOutputCols != pFieldInfo2->numOfOutputCols) { - return pFieldInfo1->numOfOutputCols - pFieldInfo2->numOfOutputCols; + if (pFieldInfo1->numOfOutput != pFieldInfo2->numOfOutput) { + return pFieldInfo1->numOfOutput - pFieldInfo2->numOfOutput; } - for (int32_t i = 0; i < pFieldInfo1->numOfOutputCols; ++i) { - TAOS_FIELD* pField1 = &pFieldInfo1->pFields[i]; - TAOS_FIELD* pField2 = &pFieldInfo2->pFields[i]; + for (int32_t i = 0; i < pFieldInfo1->numOfOutput; ++i) { + TAOS_FIELD* pField1 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo1, i); + TAOS_FIELD* pField2 = tscFieldInfoGetField((SFieldInfo*) pFieldInfo2, i); - if (pField1->type != pField2->type || pField1->bytes != pField2->bytes || + if (pField1->type != pField2->type || + pField1->bytes != pField2->bytes || strcasecmp(pField1->name, pField2->name) != 0) { return 1; } @@ -1089,99 +929,56 @@ int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2) { return 0; } -int32_t tscGetResRowLength(SQueryInfo* pQueryInfo) { - if (pQueryInfo->exprsInfo.numOfExprs <= 0) { +int32_t tscGetResRowLength(SArray* pExprList) { + size_t num = taosArrayGetSize(pExprList); + if (num == 0) { return 0; } int32_t size = 0; - for(int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { - size += pQueryInfo->exprsInfo.pExprs[i]->resBytes; + for(int32_t i = 0; i < num; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pExprList, i); + size += pExpr->resBytes; } return size; } -void tscClearFieldInfo(SFieldInfo* pFieldInfo) { +void tscFieldInfoClear(SFieldInfo* pFieldInfo) { if (pFieldInfo == NULL) { return; } - tfree(pFieldInfo->pFields); - tfree(pFieldInfo->pVisibleCols); - tfree(pFieldInfo->pSqlExpr); + taosArrayDestroy(pFieldInfo->pFields); - for(int32_t i = 0; i < pFieldInfo->numOfOutputCols; ++i) { - if (pFieldInfo->pExpr[i] != NULL) { - tExprTreeDestroy(&pFieldInfo->pExpr[i]->binExprInfo.pBinExpr, NULL); - tfree(pFieldInfo->pExpr[i]->binExprInfo.pReqColumns); - tfree(pFieldInfo->pExpr[i]); + for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { + SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, i); + + if (pInfo->pArithExprInfo != NULL) { + tExprTreeDestroy(&pInfo->pArithExprInfo->binExprInfo.pBinExpr, NULL); + tfree(pInfo->pArithExprInfo->binExprInfo.pReqColumns); } } - tfree(pFieldInfo->pExpr); + taosArrayDestroy(pFieldInfo->pSupportInfo); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } -static void _exprCheckSpace(SSqlExprInfo* pExprInfo, int32_t size) { - if (size > pExprInfo->numOfAlloc) { - uint32_t oldSize = pExprInfo->numOfAlloc; - - uint32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1U); - while (newSize < size) { - newSize = (newSize << 1U); - } - - if (newSize > TSDB_MAX_COLUMNS) { - newSize = TSDB_MAX_COLUMNS; - } - - int32_t inc = newSize - oldSize; - - pExprInfo->pExprs = realloc(pExprInfo->pExprs, newSize * sizeof(SSqlExpr)); - memset(&pExprInfo->pExprs[oldSize], 0, inc * sizeof(SSqlExpr)); - - pExprInfo->numOfAlloc = newSize; - } -} - -static void _exprEvic(SSqlExprInfo* pExprInfo, int32_t index) { - if (index < pExprInfo->numOfExprs) { - memmove(&pExprInfo->pExprs[index + 1], &pExprInfo->pExprs[index], - sizeof(pExprInfo->pExprs[0]) * (pExprInfo->numOfExprs - index)); - } -} - -SSqlExpr* tscSqlExprInsertEmpty(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId) { - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - - _exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1); - _exprEvic(pExprInfo, index); - +SSqlExpr* tscSqlExprAppend(SArray* exprInfo, int16_t functionId) { SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); pExpr->functionId = functionId; - - pExprInfo->numOfExprs++; - pExprInfo->pExprs[index] = pExpr; - return pExpr; + return taosArrayPush(exprInfo, &pExpr); } -SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, - int16_t type, int16_t size, int16_t interSize) { +SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - - _exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1); - _exprEvic(pExprInfo, index); - - SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); - pExprInfo->pExprs[index] = pExpr; - - pExpr->functionId = functionId; - int16_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); + pExpr->functionId = functionId; + // set the correct column index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; @@ -1206,21 +1003,19 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi pExpr->resBytes = size; pExpr->interResBytes = interSize; pExpr->uid = pTableMetaInfo->pTableMeta->uid; - - pExprInfo->numOfExprs++; + + taosArrayPush(pQueryInfo->exprsInfo, &pExpr); return pExpr; } SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo; - if (index > pExprInfo->numOfExprs) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, index); + if (pExpr == NULL) { return NULL; } - SSqlExpr* pExpr = pExprInfo->pExprs[index]; - pExpr->functionId = functionId; pExpr->colInfo.colIndex = srcColumnIndex; @@ -1233,7 +1028,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi } int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) { - return pQueryInfo->exprsInfo.numOfExprs; + return taosArrayGetSize(pQueryInfo->exprsInfo); } void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex) { @@ -1250,14 +1045,10 @@ void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, } SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index) { - if (pQueryInfo->exprsInfo.numOfExprs <= index) { - return NULL; - } - - return pQueryInfo->exprsInfo.pExprs[index]; + return taosArrayGetP(pQueryInfo->exprsInfo, index); } -void* tscSqlExprDestroy(SSqlExpr* pExpr) { +void* sqlExprDestroy(SSqlExpr* pExpr) { if (pExpr == NULL) { return NULL; } @@ -1274,54 +1065,46 @@ void* tscSqlExprDestroy(SSqlExpr* pExpr) { /* * NOTE: Does not release SSqlExprInfo here. */ -void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo) { - if (pExprInfo->numOfAlloc == 0) { - return; +void tscSqlExprInfoDestroy(SArray* pExprInfo) { + size_t size = taosArrayGetSize(pExprInfo); + + for(int32_t i = 0; i < size; ++i) { + SSqlExpr* pExpr = taosArrayGetP(pExprInfo, i); + sqlExprDestroy(pExpr); } - for(int32_t i = 0; i < pExprInfo->numOfExprs; ++i) { - tscSqlExprDestroy(pExprInfo->pExprs[i]); - } - - tfree(pExprInfo->pExprs); - - pExprInfo->numOfAlloc = 0; - pExprInfo->numOfExprs = 0; + taosArrayDestroy(pExprInfo); } -void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t tableuid, bool deepcopy) { - if (src == NULL || src->numOfExprs == 0) { - return; +SArray* tscSqlExprCopy(const SArray* src, uint64_t uid, bool deepcopy) { + if (src == NULL || taosArrayGetSize(src) == 0) { + return taosArrayInit(1, POINTER_BYTES); } - - *dst = *src; - - dst->pExprs = calloc(dst->numOfAlloc, POINTER_BYTES); - int16_t num = 0; - for (int32_t i = 0; i < src->numOfExprs; ++i) { - if (src->pExprs[i]->uid == tableuid) { + size_t size = taosArrayGetSize(src); + SArray* dst = taosArrayInit(size, POINTER_BYTES); + + for (int32_t i = 0; i < size; ++i) { + SSqlExpr* pExpr = taosArrayGetP(src, i); + + if (pExpr->uid == uid) { if (deepcopy) { - dst->pExprs[num] = calloc(1, sizeof(SSqlExpr)); - *dst->pExprs[num] = *src->pExprs[i]; - } else { - dst->pExprs[num] = src->pExprs[i]; - } - - num++; - } - } - - dst->numOfExprs = num; + SSqlExpr* p1 = calloc(1, sizeof(SSqlExpr)); + *p1 = *pExpr; - if (deepcopy) { - for (int32_t i = 0; i < dst->numOfExprs; ++i) { - for (int32_t j = 0; j < src->pExprs[i]->numOfParams; ++j) { - tVariantAssign(&dst->pExprs[i]->param[j], &src->pExprs[i]->param[j]); + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { + tVariantAssign(&p1->param[j], &pExpr->param[j]); + } + + taosArrayPush(dst, &p1); + } else { + taosArrayPush(dst, &pExpr); } } } + + return dst; } SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { @@ -1646,8 +1429,9 @@ void tscTagCondRelease(STagCond* pTagCond) { void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); pColInfo[i].functionId = pExpr->functionId; @@ -1707,7 +1491,7 @@ void tscCleanSqlCmd(SSqlCmd* pCmd) { * * If connection need to be recycled, the SqlObj also should be freed. */ -bool tscShouldFreeAsyncSqlObj(SSqlObj* pSql) { +bool tscShouldBeFreed(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql || pSql->fp == NULL) { return false; } @@ -1783,7 +1567,7 @@ int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQuer return TSDB_CODE_SUCCESS; } -STableMetaInfo* tscGetMeterMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { +STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { int32_t k = -1; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { @@ -1813,6 +1597,13 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { pCmd->pQueryInfo = (SQueryInfo**)tmp; SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + + // todo refactor to extract functions. + pQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); + pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); + + pQueryInfo->exprsInfo = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; @@ -1821,9 +1612,9 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { tscTagCondRelease(&pQueryInfo->tagCond); - tscClearFieldInfo(&pQueryInfo->fieldsInfo); + tscFieldInfoClear(&pQueryInfo->fieldsInfo); - tscSqlExprInfoDestroy(&pQueryInfo->exprsInfo); + tscSqlExprInfoDestroy(pQueryInfo->exprsInfo); memset(&pQueryInfo->exprsInfo, 0, sizeof(pQueryInfo->exprsInfo)); tscColumnListDestroy(pQueryInfo->colList); @@ -1851,7 +1642,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); doClearSubqueryInfo(pQueryInfo); - tscRemoveAllMeterMetaInfo(pQueryInfo, (const char*)addr, false); + tscClearAllTableMetaInfo(pQueryInfo, (const char*)addr, false); tfree(pQueryInfo); } @@ -1919,7 +1710,7 @@ void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro pQueryInfo->numOfTables -= 1; } -void tscRemoveAllMeterMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { +void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { tscTrace("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); int32_t index = pQueryInfo->numOfTables; @@ -1937,7 +1728,6 @@ void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); tfree(pTableMetaInfo->vgroupList); -// taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pMetricMeta), removeFromCache); } void tscResetForNextRetrieve(SSqlRes* pRes) { @@ -2003,8 +1793,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(int64_t)); - memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(int64_t)); + pNewQueryInfo->defaultVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); + memcpy(pNewQueryInfo->defaultVal, pQueryInfo->defaultVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); } if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { @@ -2024,30 +1814,33 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } uint64_t uid = pTableMetaInfo->pTableMeta->uid; - tscSqlExprCopy(&pNewQueryInfo->exprsInfo, &pQueryInfo->exprsInfo, uid, true); + pNewQueryInfo->exprsInfo = tscSqlExprCopy(pQueryInfo->exprsInfo, uid, true); - int32_t numOfOutputCols = pNewQueryInfo->exprsInfo.numOfExprs; + int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo); - if (numOfOutputCols > 0) { - int32_t* indexList = calloc(1, numOfOutputCols * sizeof(int32_t)); - for (int32_t i = 0, j = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { + if (numOfOutput > 0) { + int32_t* indexList = calloc(1, numOfOutput * sizeof(int32_t)); + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0, j = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->uid == uid) { indexList[j++] = i; } } - tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pNewQueryInfo->fieldsInfo, indexList, numOfOutputCols); + tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pNewQueryInfo->fieldsInfo, indexList, numOfOutput); free(indexList); // make sure the the sqlExpr for each fields is correct // todo handle the agg arithmetic expression - for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutputCols; ++f) { - char* name = pNewQueryInfo->fieldsInfo.pFields[f].name; - for(int32_t k1 = 0; k1 < pNewQueryInfo->exprsInfo.numOfExprs; ++k1) { + for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { + TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); + for(int32_t k1 = 0; k1 < numOfExprs; ++k1) { SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); - if (strcmp(name, pExpr1->aliasName) == 0) { - pNewQueryInfo->fieldsInfo.pSqlExpr[f] = pExpr1; + if (strcmp(field->name, pExpr1->aliasName) == 0) { + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); + pInfo->pSqlExpr = pExpr1; } } } @@ -2058,15 +1851,6 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->fp = fp; pNew->param = param; - char key[TSDB_MAX_TAGS_LEN + 1] = {0}; - if (cmd == TSDB_SQL_SELECT) { - tscGetMetricMetaCacheKey(pQueryInfo, key, uid); - } - -#ifdef _DEBUG_VIEW - tscTrace("the metricmeta key is:%s", key); -#endif - char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; @@ -2097,8 +1881,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void tscTrace( "%p new subquery: %p, tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, - pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, pNewQueryInfo->exprsInfo.numOfExprs, - size, pNewQueryInfo->fieldsInfo.numOfOutputCols, pFinalInfo->name, pNewQueryInfo->stime, + pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), + size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->stime, pNewQueryInfo->etime, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); tscPrintSelectClause(pNew, 0); @@ -2326,3 +2110,11 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { tscProcessSql(pSql); } } + +char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column) { + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, column); + + return ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows; +} + diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index bd8d4d09f4..7d9c343a85 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -372,7 +372,7 @@ typedef struct SColIndex { /* sql function msg, to describe the message to vnode about sql function * operations in select clause */ -typedef struct SSqlFuncExprMsg { +typedef struct SSqlFuncMsg { int16_t functionId; int16_t numOfParams; @@ -386,21 +386,21 @@ typedef struct SSqlFuncExprMsg { char * pz; } argValue; } arg[3]; -} SSqlFuncExprMsg; +} SSqlFuncMsg; -typedef struct SSqlBinaryExprInfo { +typedef struct SExprInfo { struct tExprNode *pBinExpr; /* for binary expression */ int32_t numOfCols; /* binary expression involves the readed number of columns*/ SColIndex * pReqColumns; /* source column list */ -} SSqlBinaryExprInfo; +} SExprInfo; -typedef struct SSqlFunctionExpr { - SSqlFuncExprMsg pBase; - SSqlBinaryExprInfo binExprInfo; - int16_t resBytes; - int16_t resType; - int16_t interResBytes; -} SSqlFunctionExpr; +typedef struct SArithExprInfo { + SSqlFuncMsg pBase; + SExprInfo binExprInfo; + int16_t bytes; + int16_t type; + int16_t interResBytes; +} SArithExprInfo; typedef struct SColumnFilterInfo { int16_t lowerRelOptr; @@ -469,7 +469,7 @@ typedef struct { int64_t limit; int64_t offset; uint16_t queryType; // denote another query process - int16_t numOfOutputCols; // final output columns numbers + int16_t numOfOutput; // final output columns numbers int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list From 79f3d77852125790606ac2fc85bf2b2cac95b1a4 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 23 Apr 2020 12:23:30 +0800 Subject: [PATCH 05/63] [td-171]fix bug in column values filter query --- src/query/inc/queryExecutor.h | 4 +- src/query/inc/tsqlfunction.h | 2 +- src/query/src/queryExecutor.c | 316 ++++++++++++++++------------------ src/query/src/queryUtil.c | 8 +- 4 files changed, 151 insertions(+), 179 deletions(-) diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 504d20f992..a9a9424a7c 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -122,13 +122,13 @@ typedef struct SQuery { int64_t slidingTime; // sliding time for sliding window query char slidingTimeUnit; // interval data type, used for daytime revise int8_t precision; - int16_t numOfOutputCols; + int16_t numOfOutput; int16_t interpoType; int16_t checkBuffer; // check if the buffer is full during scan each block SLimitVal limit; int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; - SSqlFunctionExpr* pSelectExpr; + SArithExprInfo* pSelectExpr; SColumnInfo* colList; int32_t numOfFilterCols; int64_t* defaultVal; diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 3cfcf83268..54055e9d33 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -114,7 +114,7 @@ enum { #define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) typedef struct SArithmeticSupport { - SSqlFunctionExpr *pExpr; + SArithExprInfo *pArithExpr; int32_t elemSize[TSDB_MAX_COLUMNS]; int32_t numOfCols; int32_t offset; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index aefe4a3b8b..02580e6e87 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -251,26 +251,12 @@ bool doFilterData(SQuery *pQuery, int32_t elemPos) { return true; } -bool vnodeFilterData(SQuery *pQuery, int32_t *numOfActualRead, int32_t index) { - (*numOfActualRead)++; - if (!doFilterData(pQuery, index)) { - return false; - } - - if (pQuery->limit.offset > 0) { - pQuery->limit.offset--; // ignore this qualified row - return false; - } - - return true; -} - int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; bool hasMainFunction = hasMainOutput(pQuery); int64_t maxOutput = 0; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; /* @@ -347,7 +333,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { bool hasTags = false; int32_t numOfSelectivity = 0; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { hasTags = true; @@ -368,7 +354,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS_COMP; } -bool limitResults(SQInfo *pQInfo) { +static bool limitResults(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { @@ -383,7 +369,7 @@ bool limitResults(SQInfo *pQInfo) { } static bool isTopBottomQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS) { continue; @@ -730,7 +716,7 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; pCtx[k].nStartQueryTimestamp = pWin->skey; @@ -754,7 +740,7 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].nStartQueryTimestamp = pWin->skey; int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; @@ -839,7 +825,7 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 int32_t functionId = pQuery->pSelectExpr[col].pBase.functionId; if (functionId == TSDB_FUNC_ARITHM) { - sas->pExpr = &pQuery->pSelectExpr[col]; + sas->pArithExpr = &pQuery->pSelectExpr[col]; // set the start offset to be the lowest start position, no matter asc/desc query order if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -907,9 +893,9 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * primaryKeyCol = (TSKEY *)(pColInfo->pData); } - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; SDataStatis *tpField = NULL; @@ -966,7 +952,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * * since the selectivity + tag_prj query needs all parameters been set done. * tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY */ - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunction(&pCtx[k]); @@ -1098,7 +1084,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS TSKEY *primaryKeyCol = (TSKEY*) ((SColumnInfoData *)taosArrayGet(pDataBlock, 0))->pData; bool groupbyStateValue = isGroupbyNormalCol(pQuery->pGroupbyExpr); - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutputCols, sizeof(SArithmeticSupport)); + SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); int16_t type = 0; int16_t bytes = 0; @@ -1109,7 +1095,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // groupbyColumnData = getGroupbyColumnData(pQuery, data, &type, &bytes); } - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; SDataStatis *pColStatis = NULL; @@ -1218,7 +1204,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // all startOffset are identical offset -= pCtx[0].startOffset; - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunctionF(&pCtx[k], offset); @@ -1350,9 +1336,9 @@ static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) { int16_t tagLen = 0; - SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutputCols, POINTER_BYTES); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; @@ -1374,7 +1360,7 @@ static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) { } static void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { setResultInfoBuf(&pResultInfo[i], pQuery->pSelectExpr[i].interResBytes, isStableQuery); } } @@ -1383,16 +1369,16 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order qTrace("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; - pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); - pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutputCols, sizeof(SQLFunctionCtx)); + pRuntimeEnv->resultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); + pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { goto _error_clean; } pRuntimeEnv->offset[0] = 0; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); @@ -1400,8 +1386,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->ptsOutputBuf = NULL; - pCtx->outputBytes = pQuery->pSelectExpr[i].resBytes; - pCtx->outputType = pQuery->pSelectExpr[i].resType; + pCtx->outputBytes = pQuery->pSelectExpr[i].bytes; + pCtx->outputType = pQuery->pSelectExpr[i].type; pCtx->order = pQuery->order.order; pCtx->functionId = pSqlFuncMsg->functionId; @@ -1463,10 +1449,10 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; qTrace("QInfo:%p teardown runtime env", GET_QINFO_ADDR(pQuery)); - cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutputCols); + cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo, pQuery->numOfOutput); if (pRuntimeEnv->pCtx != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; for (int32_t j = 0; j < pCtx->numOfParams; ++j) { @@ -1485,7 +1471,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { taosDestoryInterpoInfo(&pRuntimeEnv->interpoInfo); if (pRuntimeEnv->pInterpoBuf != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { tfree(pRuntimeEnv->pInterpoBuf[i]); } @@ -1528,8 +1514,8 @@ bool isFixedOutputQuery(SQuery *pQuery) { return true; } - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; // ignore the ts_comp function if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 && @@ -1550,7 +1536,7 @@ bool isFixedOutputQuery(SQuery *pQuery) { } bool isPointInterpoQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { return true; @@ -1562,7 +1548,7 @@ bool isPointInterpoQuery(SQuery *pQuery) { // TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION bool isSumAvgRateQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS) { continue; @@ -1578,7 +1564,7 @@ bool isSumAvgRateQuery(SQuery *pQuery) { } bool isFirstLastRowQuery(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; if (functionID == TSDB_FUNC_LAST_ROW) { return true; @@ -1594,7 +1580,7 @@ bool notHasQueryTimeRange(SQuery *pQuery) { } static bool needReverseScan(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { continue; @@ -1767,8 +1753,8 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { pQuery->checkBuffer = 0; } else { bool hasMultioutput = false; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlFuncExprMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { continue; } @@ -1800,7 +1786,7 @@ bool vnodeParametersSafetyCheck(SQuery *pQuery) { // todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which // the scan order is not matter static bool onlyOneQueryType(SQuery *pQuery, int32_t functId, int32_t functIdDst) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || @@ -1977,7 +1963,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI if (key == pQuery->window.skey) { // the queried timestamp has value, return it directly without interpolation - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { tVariantCreateFromBinary(&pRuntimeEnv->pCtx[i].param[3], (char *)&count, sizeof(count), TSDB_DATA_TYPE_INT); pRuntimeEnv->pCtx[i].param[0].i64Key = key; @@ -1988,7 +1974,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI count = 2; if (pQuery->interpoType == TSDB_INTERPO_SET_VALUE) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; // only the function of interp needs the corresponding information @@ -2023,7 +2009,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI TSKEY prevKey = *(TSKEY *)pPointInterpSupport->pPrevPoint[0]; TSKEY nextKey = *(TSKEY *)pPointInterpSupport->pNextPoint[0]; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; // tag column does not need the interp environment @@ -2107,11 +2093,11 @@ static UNUSED_FUNC void allocMemForInterpo(SQInfo *pQInfo, SQuery *pQuery, void assert(isIntervalQuery(pQuery) || (pQuery->intervalTime == 0 && isPointInterpoQuery(pQuery))); if (isIntervalQuery(pQuery)) { - pQInfo->runtimeEnv.pInterpoBuf = malloc(POINTER_BYTES * pQuery->numOfOutputCols); + pQInfo->runtimeEnv.pInterpoBuf = malloc(POINTER_BYTES * pQuery->numOfOutput); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { pQInfo->runtimeEnv.pInterpoBuf[i] = - calloc(1, sizeof(tFilePage) + pQuery->pSelectExpr[i].resBytes * pMeterObj->pointsPerFileBlock); + calloc(1, sizeof(tFilePage) + pQuery->pSelectExpr[i].bytes * pMeterObj->pointsPerFileBlock); } } } @@ -2162,7 +2148,7 @@ char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWi int32_t realRowId = pResult->pos.rowId * getRowParamForMultiRowsOutput(pQuery, pRuntimeEnv->stableQuery); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * numOfRows + - pQuery->pSelectExpr[columnIndex].resBytes * realRowId; + pQuery->pSelectExpr[columnIndex].bytes * realRowId; } /** @@ -2206,7 +2192,7 @@ UNUSED_FUNC void vnodeDecMeterRefcnt(SQInfo *pQInfo) { UNUSED_FUNC void setTimestampRange(SQueryRuntimeEnv *pRuntimeEnv, int64_t stime, int64_t etime) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_SPREAD) { @@ -2265,7 +2251,7 @@ static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFun } // todo disable this opt code block temporarily - // for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + // for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { // int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; // if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { // return top_bot_datablock_filter(&pCtx[i], functId, (char *)&pField[i].min, (char *)&pField[i].max); @@ -2293,7 +2279,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl if (pQuery->numOfFilterCols > 0) { r = BLK_DATA_ALL_NEEDED; } else { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t colId = pQuery->pSelectExpr[i].pBase.colInfo.colId; r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); @@ -2446,8 +2432,8 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { int32_t remain = pRec->capacity - pRec->rows; int32_t newSize = pRec->capacity + (blockInfo.rows - remain); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - int32_t bytes = pQuery->pSelectExpr[i].resBytes; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t bytes = pQuery->pSelectExpr[i].bytes; char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(SData)); if (tmp == NULL) { // todo handle the oom @@ -2519,13 +2505,13 @@ static void doSetTagValueInParam(void *tsdb, STableId id, int32_t tagColId, tVar void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void *tsdb) { SQuery *pQuery = pRuntimeEnv->pQuery; - SSqlFuncExprMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; - if (pQuery->numOfOutputCols == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { + SSqlFuncMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; + if (pQuery->numOfOutput == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { assert(pFuncMsg->numOfParams == 1); doSetTagValueInParam(tsdb, id, pFuncMsg->arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag); } else { // set tag value, by which the results are aggregated. - for (int32_t idx = 0; idx < pQuery->numOfOutputCols; ++idx) { + for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { SColIndex *pCol = &pQuery->pSelectExpr[idx].pBase.colInfo; // ts_comp column required the tag value for join filter @@ -2551,7 +2537,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (!mergeFlag) { pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; @@ -2575,7 +2561,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes } } - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId == TSDB_FUNC_TAG_DUMMY) { continue; @@ -2656,7 +2642,7 @@ static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOfRows) { #if 0 - int32_t numOfCols = pQuery->numOfOutputCols; + int32_t numOfCols = pQuery->numOfOutput; printf("super table query intermediate result, total:%d\n", numOfRows); SQInfo * pQInfo = (SQInfo *)(GET_QINFO_ADDR(pQuery)); @@ -2664,32 +2650,32 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { - switch (pQuery->pSelectExpr[i].resType) { + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { int32_t colIndex = pQuery->pSelectExpr[i].pBase.colInfo.colIndex; int32_t type = 0; if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].pBase.colInfo.flag)) { - type = pQuery->pSelectExpr[i].resType; + type = pQuery->pSelectExpr[i].type; } else { type = pMeterObj->schema[colIndex].type; } - printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j, + printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j, type); break; } case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: - printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%" PRId64 "\t", *(int64_t *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; case TSDB_DATA_TYPE_INT: - printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%d\t", *(int32_t *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; case TSDB_DATA_TYPE_FLOAT: - printf("%f\t", *(float *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%f\t", *(float *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; case TSDB_DATA_TYPE_DOUBLE: - printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pSelectExpr[i].resBytes * j)); + printf("%lf\t", *(double *)(pdata[i]->data + pQuery->pSelectExpr[i].bytes * j)); break; } } @@ -2807,7 +2793,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { for (int32_t num = 0; num < list.size; ++num) { tFilePage *pData = getResultBufferPageById(pResultBuf, list.pData[num]); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; @@ -2828,7 +2814,7 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pW SQuery *pQuery = pRuntimeEnv->pQuery; int64_t maxOutput = 0; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; /* @@ -2885,7 +2871,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SLoserTreeInfo *pTree = NULL; tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - SResultInfo *pResultInfo = calloc(pQuery->numOfOutputCols, sizeof(SResultInfo)); + SResultInfo *pResultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); @@ -2971,7 +2957,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { tfree(posList); pQInfo->offset = 0; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { tfree(pResultInfo[i].interResultBuf); } @@ -3002,7 +2988,7 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); // pagewise copy to dest buffer - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; buf->numOfElems = r; @@ -3019,7 +3005,7 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { } void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; @@ -3044,7 +3030,7 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo * SWindowResult *buf = getWindowResult(pWindowResInfo, i); // open/close the specified query for each group result - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || @@ -3066,7 +3052,7 @@ void disableFuncInReverseScan(SQueryRuntimeEnv *pRuntimeEnv) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || isIntervalQuery(pQuery)) { doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); } else { // for simple result of table query, - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; @@ -3085,7 +3071,7 @@ void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { pRuntimeEnv->pCtx[i].order = (pRuntimeEnv->pCtx[i].order) ^ 1u; } @@ -3106,14 +3092,14 @@ void disableFuncForReverseScan(SQInfo *pQInfo, int32_t order) { void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SWITCH_ORDER(pRuntimeEnv->pCtx[i] .order); // = (pRuntimeEnv->pCtx[i].order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; } } void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo) { - int32_t numOfCols = pQuery->numOfOutputCols; + int32_t numOfCols = pQuery->numOfOutput; pResultRow->resultInfo = calloc((size_t)numOfCols, sizeof(SResultInfo)); pResultRow->pos = *posInfo; @@ -3125,7 +3111,7 @@ void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTa void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->aOutputBuf = pQuery->sdata[i]->data; @@ -3142,7 +3128,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].resBytes * pQuery->rec.capacity); + memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity); } initCtxOutputBuf(pRuntimeEnv); @@ -3152,7 +3138,7 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { SQuery *pQuery = pRuntimeEnv->pQuery; // reset the execution contexts - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; assert(functionId != TSDB_FUNC_DIFF); @@ -3179,7 +3165,7 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; pRuntimeEnv->pCtx[j].currentStage = 0; @@ -3205,7 +3191,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfSkip = (int32_t) pQuery->limit.offset; pQuery->rec.rows -= numOfSkip; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; @@ -3247,7 +3233,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { setWindowResOutputBuf(pRuntimeEnv, pResult); - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; if (functId == TSDB_FUNC_TS) { continue; @@ -3260,7 +3246,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } } } else { - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; if (functId == TSDB_FUNC_TS) { continue; @@ -3428,7 +3414,7 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { setWindowResOutputBuf(pRuntimeEnv, buf); - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); } @@ -3440,14 +3426,14 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { } } else { - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); } } } static bool hasMainOutput(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) { @@ -3550,7 +3536,7 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->numOfElems == 0, there is only fixed number of results for each group - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); @@ -3651,7 +3637,7 @@ void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSK } bool requireTimestamp(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; i++) { + for (int32_t i = 0; i < pQuery->numOfOutput; i++) { int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_NEED_TS) != 0) { return true; @@ -3732,7 +3718,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde pQInfo->groupIndex += 1; } - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; @@ -3855,18 +3841,18 @@ static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, t assert(pRuntimeEnv->pCtx[0].outputBytes == TSDB_KEYSIZE); // build support structure for performing interpolation - SSchema *pSchema = calloc(1, sizeof(SSchema) * pQuery->numOfOutputCols); - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + SSchema *pSchema = calloc(1, sizeof(SSchema) * pQuery->numOfOutput); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { pSchema[i].bytes = pRuntimeEnv->pCtx[i].outputBytes; - pSchema[i].type = pQuery->pSelectExpr[i].resType; + pSchema[i].type = pQuery->pSelectExpr[i].type; } -// SColumnModel *pModel = createColumnModel(pSchema, pQuery->numOfOutputCols, pQuery->pointsToRead); +// SColumnModel *pModel = createColumnModel(pSchema, pQuery->numOfOutput, pQuery->pointsToRead); char * srcData[TSDB_MAX_COLUMNS] = {0}; int32_t functions[TSDB_MAX_COLUMNS] = {0}; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { srcData[i] = pDataSrc[i]->data; functions[i] = pQuery->pSelectExpr[i].pBase.functionId; } @@ -3884,8 +3870,8 @@ static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, t static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { - int32_t bytes = pQuery->pSelectExpr[col].resBytes; + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + int32_t bytes = pQuery->pSelectExpr[col].bytes; memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); data += bytes * numOfRows; @@ -3922,9 +3908,9 @@ int32_t vnodeQueryResultInterpolate(SQInfo *pQInfo, tFilePage **pDst, tFilePage ret -= pQuery->limit.offset; // todo !!!!there exactly number of interpo is not valid. // todo refactor move to the beginning of buffer - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].resBytes * pQuery->limit.offset, - ret * pQuery->pSelectExpr[i].resBytes); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset, + ret * pQuery->pSelectExpr[i].bytes); } pQuery->limit.offset = 0; return ret; @@ -4302,7 +4288,7 @@ static UNUSED_FUNC bool doCheckWithPrevQueryRange(SQuery *pQuery, TSKEY nextKey) static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); if (pResInfo != NULL) { pResInfo->complete = false; @@ -4672,7 +4658,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pStatus->closed = true; // enable return all results for group by normal columns SWindowResult *pResult = &pWindowResInfo->pResult[i]; - for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); } } @@ -5018,8 +5004,8 @@ static void tableIntervalProcess(SQInfo *pQInfo) { taosInterpoSetStartInfo(&pRuntimeEnv->interpoInfo, pQuery->rec.rows, pQuery->interpoType); SData **pInterpoBuf = pRuntimeEnv->pInterpoBuf; - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.rows * pQuery->pSelectExpr[i].resBytes); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + memcpy(pInterpoBuf[i]->data, pQuery->sdata[i]->data, pQuery->rec.rows * pQuery->pSelectExpr[i].bytes); } numOfInterpo = 0; @@ -5159,7 +5145,7 @@ static void stableQueryImpl(SQInfo *pQInfo) { sem_post(&pQInfo->dataReady); } -static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { +static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg) { int32_t j = 0; while (j < pQueryMsg->numOfCols) { @@ -5173,7 +5159,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg return j; } -bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncExprMsg *pExprMsg) { +bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg) { int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg); return j < pQueryMsg->numOfCols; } @@ -5199,8 +5185,8 @@ static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { return -1; } - if (pQueryMsg->numOfOutputCols > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutputCols <= 0) { - qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutputCols); + if (pQueryMsg->numOfOutput > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutput <= 0) { + qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutput); return -1; } @@ -5244,7 +5230,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @param pExpr * @return */ -static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncExprMsg ***pExpr, +static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, char **tagCond, SColIndex **groupbyCols) { pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); @@ -5260,7 +5246,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->queryType = htons(pQueryMsg->queryType); pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); - pQueryMsg->numOfOutputCols = htons(pQueryMsg->numOfOutputCols); + pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); @@ -5316,10 +5302,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, bool hasArithmeticFunction = false; - *pExpr = calloc(pQueryMsg->numOfOutputCols, POINTER_BYTES); - SSqlFuncExprMsg *pExprMsg = (SSqlFuncExprMsg *)pMsg; + *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { (*pExpr)[i] = pExprMsg; pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); @@ -5328,7 +5314,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pExprMsg->functionId = htons(pExprMsg->functionId); pExprMsg->numOfParams = htons(pExprMsg->numOfParams); - pMsg += sizeof(SSqlFuncExprMsg); + pMsg += sizeof(SSqlFuncMsg); for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { pExprMsg->arg[j].argType = htons(pExprMsg->arg[j].argType); @@ -5355,7 +5341,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } } - pExprMsg = (SSqlFuncExprMsg *)pMsg; + pExprMsg = (SSqlFuncMsg *)pMsg; } pQueryMsg->colNameLen = htonl(pQueryMsg->colNameLen); @@ -5393,11 +5379,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->defaultVal = (uint64_t)(pMsg); int64_t *v = (int64_t *)pMsg; - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { v[i] = htobe64(v[i]); } - pMsg += sizeof(int64_t) * pQueryMsg->numOfOutputCols; + pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; } // the tag query condition expression string is located at the end of query msg @@ -5411,14 +5397,14 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 ", offset:%" PRId64, pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, - pQueryMsg->order, pQueryMsg->numOfOutputCols, pQueryMsg->numOfCols, pQueryMsg->intervalTime, + pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType, pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset); return 0; } -static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMsg *pQueryMsg) { -// SSqlBinaryExprInfo *pBinaryExprInfo = &pExpr->binExprInfo; +static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pExpr, SQueryTableMsg *pQueryMsg) { +// SExprInfo *pBinaryExprInfo = &pExpr->binExprInfo; // SColumnInfo * pColMsg = pQueryMsg->colList; #if 0 tExprNode* pBinExpr = NULL; @@ -5468,12 +5454,12 @@ static int32_t buildAirthmeticExprFromMsg(SSqlFunctionExpr *pExpr, SQueryTableMs return TSDB_CODE_SUCCESS; } -static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunctionExpr **pSqlFuncExpr, - SSqlFuncExprMsg **pExprMsg) { +static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExprInfo **pSqlFuncExpr, + SSqlFuncMsg **pExprMsg) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; - SSqlFunctionExpr *pExprs = (SSqlFunctionExpr *)calloc(1, sizeof(SSqlFunctionExpr) * pQueryMsg->numOfOutputCols); + SArithExprInfo *pExprs = (SArithExprInfo *)calloc(1, sizeof(SArithExprInfo) * pQueryMsg->numOfOutput); if (pExprs == NULL) { return TSDB_CODE_SERV_OUT_OF_MEMORY; } @@ -5481,9 +5467,9 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); int16_t tagLen = 0; - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { pExprs[i].pBase = *pExprMsg[i]; - pExprs[i].resBytes = 0; + pExprs[i].bytes = 0; int16_t type = 0; int16_t bytes = 0; @@ -5509,22 +5495,22 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct } int32_t param = pExprs[i].pBase.arg[0].argValue.i64; - if (getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].resType, &pExprs[i].resBytes, + if (getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { tfree(pExprs); return TSDB_CODE_INVALID_QUERY_MSG; } if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].pBase.functionId == TSDB_FUNC_TS_DUMMY) { - tagLen += pExprs[i].resBytes; + tagLen += pExprs[i].bytes; } - assert(isValidDataType(pExprs[i].resType, pExprs[i].resBytes)); + assert(isValidDataType(pExprs[i].type, pExprs[i].bytes)); } // get the correct result size for top/bottom query, according to the number of tags columns in selection clause // TODO refactor - for (int32_t i = 0; i < pQueryMsg->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { pExprs[i].pBase = *pExprMsg[i]; int16_t functId = pExprs[i].pBase.functionId; if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { @@ -5537,7 +5523,7 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SSqlFunct int32_t ret = getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, pExprs[i].pBase.arg[0].argValue.i64, - &pExprs[i].resType, &pExprs[i].resBytes, &pExprs[i].interResBytes, tagLen, isSuperTable); + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, tagLen, isSuperTable); assert(ret == TSDB_CODE_SUCCESS); } } @@ -5655,23 +5641,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { static void doUpdateExprColumnIndex(SQuery *pQuery) { assert(pQuery->pSelectExpr != NULL && pQuery != NULL); - // int32_t i = 0, j = 0; - // while (i < pQuery->numOfCols && j < pMeterObj->numOfColumns) { - // if (pQuery->colList[i].data.colId == pMeterObj->schema[j].colId) { - // pQuery->colList[i++].colIndex = (int16_t)j++; - // } else if (pQuery->colList[i].data.colId < pMeterObj->schema[j].colId) { - // pQuery->colList[i++].colIndex = -1; - // } else if (pQuery->colList[i].data.colId > pMeterObj->schema[j].colId) { - // j++; - // } - // } - // while (i < pQuery->numOfCols) { - // pQuery->colList[i++].colIndex = -1; // not such column in current meter - // } - - for (int32_t k = 0; k < pQuery->numOfOutputCols; ++k) { - SSqlFuncExprMsg *pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + SSqlFuncMsg *pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) { continue; } @@ -5686,7 +5658,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } } -static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pExprs, +static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SArithExprInfo *pExprs, STableGroupInfo *groupInfo) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { @@ -5697,10 +5669,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->runtimeEnv.pQuery = pQuery; int16_t numOfCols = pQueryMsg->numOfCols; - int16_t numOfOutputCols = pQueryMsg->numOfOutputCols; + int16_t numOfOutput = pQueryMsg->numOfOutput; pQuery->numOfCols = numOfCols; - pQuery->numOfOutputCols = numOfOutputCols; + pQuery->numOfOutput = numOfOutput; pQuery->limit.limit = pQueryMsg->limit; pQuery->limit.offset = pQueryMsg->offset; pQuery->order.order = pQueryMsg->order; @@ -5725,9 +5697,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // calculate the result row size - for (int16_t col = 0; col < numOfOutputCols; ++col) { - assert(pExprs[col].resBytes > 0); - pQuery->rowSize += pExprs[col].resBytes; + for (int16_t col = 0; col < numOfOutput; ++col) { + assert(pExprs[col].bytes > 0); + pQuery->rowSize += pExprs[col].bytes; } doUpdateExprColumnIndex(pQuery); @@ -5738,7 +5710,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // prepare the result buffer - pQuery->sdata = (SData **)calloc(pQuery->numOfOutputCols, POINTER_BYTES); + pQuery->sdata = (SData **)calloc(pQuery->numOfOutput, POINTER_BYTES); if (pQuery->sdata == NULL) { goto _cleanup; } @@ -5747,11 +5719,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->rec.capacity = 4096; pQuery->rec.threshold = 4000; - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { - assert(pExprs[col].interResBytes >= pExprs[col].resBytes); + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + assert(pExprs[col].interResBytes >= pExprs[col].bytes); // allocate additional memory for interResults that are usually larger then final results - size_t size = (pQuery->rec.capacity + 1) * pExprs[col].resBytes + pExprs[col].interResBytes + sizeof(SData); + size_t size = (pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interResBytes + sizeof(SData); pQuery->sdata[col] = (SData *)calloc(1, size); if (pQuery->sdata[col] == NULL) { goto _cleanup; @@ -5759,13 +5731,13 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } if (pQuery->interpoType != TSDB_INTERPO_NONE) { - pQuery->defaultVal = malloc(sizeof(int64_t) * pQuery->numOfOutputCols); + pQuery->defaultVal = malloc(sizeof(int64_t) * pQuery->numOfOutput); if (pQuery->defaultVal == NULL) { goto _cleanup; } // the first column is the timestamp - memcpy(pQuery->defaultVal, (char *)pQueryMsg->defaultVal, pQuery->numOfOutputCols * sizeof(int64_t)); + memcpy(pQuery->defaultVal, (char *)pQueryMsg->defaultVal, pQuery->numOfOutput * sizeof(int64_t)); } // to make sure third party won't overwrite this structure @@ -5792,7 +5764,7 @@ _cleanup: tfree(pQuery->defaultVal); if (pQuery->sdata != NULL) { - for (int16_t col = 0; col < pQuery->numOfOutputCols; ++col) { + for (int16_t col = 0; col < pQuery->numOfOutput; ++col) { tfree(pQuery->sdata[col]); } } @@ -5872,7 +5844,7 @@ static void freeQInfo(SQInfo *pQInfo) { setQueryKilled(pQInfo); qTrace("QInfo:%p start to free QInfo", pQInfo); - for (int32_t col = 0; col < pQuery->numOfOutputCols; ++col) { + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { tfree(pQuery->sdata[col]); } @@ -5891,8 +5863,8 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->sdata); if (pQuery->pSelectExpr != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { - SSqlBinaryExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; if (pBinExprInfo->numOfCols > 0) { tfree(pBinExprInfo->pReqColumns); @@ -5988,7 +5960,7 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) char * tagCond = NULL; SArray * pTableIdList = NULL; - SSqlFuncExprMsg **pExprMsg = NULL; + SSqlFuncMsg **pExprMsg = NULL; SColIndex * pGroupColIndex = NULL; if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &pGroupColIndex)) != TSDB_CODE_SUCCESS) { @@ -6007,7 +5979,7 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) goto _query_over; } - SSqlFunctionExpr *pExprs = NULL; + SArithExprInfo *pExprs = NULL; if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { goto _query_over; } diff --git a/src/query/src/queryUtil.c b/src/query/src/queryUtil.c index 17410b2868..b4d8911284 100644 --- a/src/query/src/queryUtil.c +++ b/src/query/src/queryUtil.c @@ -217,11 +217,11 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow return; } - for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutputCols; ++i) { + for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes); - size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memset(s, 0, size); resetResultInfo(pResultInfo); @@ -245,7 +245,7 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con dst->window = src->window; dst->status = src->status; - int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutputCols; + int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; for (int32_t i = 0; i < nOutputCols; ++i) { SResultInfo *pDst = &dst->resultInfo[i]; @@ -261,7 +261,7 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con // copy the output buffer data from src to dst, the position info keep unchanged char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst); char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src); - size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].resBytes; + size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memcpy(dstBuf, srcBuf, s); } From 5e221d723124f8223ddb6b6139f9c95fd27494cc Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 23 Apr 2020 14:00:46 +0800 Subject: [PATCH 06/63] [td-171] refactor codes --- src/client/inc/tscUtil.h | 15 +- src/client/inc/tsclient.h | 255 ++++++++++++++--------------- src/client/src/tscAsync.c | 2 +- src/client/src/tscLocal.c | 2 +- src/client/src/tscParseInsert.c | 2 +- src/client/src/tscSQLParser.c | 10 +- src/client/src/tscSchemaUtil.c | 2 +- src/client/src/tscSecondaryMerge.c | 4 +- src/client/src/tscServer.c | 4 +- src/client/src/tscSql.c | 2 +- src/client/src/tscStream.c | 2 +- src/client/src/tscSubquery.c | 6 +- src/client/src/tscUtil.c | 48 ++---- src/mnode/src/mgmtTable.c | 6 +- 14 files changed, 168 insertions(+), 192 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 723e5bf9bc..f366277e27 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -69,8 +69,9 @@ typedef struct SJoinSubquerySupporter { int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); -void tscDestroyDataBlock(STableDataBlocks* pDataBlock); +void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); +void tscDestroyDataBlock(STableDataBlocks* pDataBlock); +void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, uint32_t offset); @@ -133,9 +134,8 @@ SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FI SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index); TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); -void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo); -void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size); -void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src); +void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); +void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src); void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo); int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); @@ -235,11 +235,6 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex); int16_t tscGetJoinTagColIndexByUid(STagCond* pTagCond, uint64_t uid); -TAOS* taos_connect_a(char* ip, char* user, char* pass, char* db, uint16_t port, void (*fp)(void*, TAOS_RES*, int), - void* param, void** taos); - -void sortRemoveDuplicates(STableDataBlocks* dataBuf); - void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 59cba3105f..89bed123c4 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -22,20 +22,28 @@ extern "C" { #include "os.h" -#include "qsqlparser.h" -#include "qsqltype.h" -#include "qtsbuf.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tglobal.h" -#include "tutil.h" #include "tsqlfunction.h" +#include "tutil.h" + +#include "qsqlparser.h" +#include "qsqltype.h" +#include "qtsbuf.h" #include "queryExecutor.h" // forward declaration struct SSqlInfo; +struct SLocalReducer; + +// data source from sql string or from file +enum { + DATA_FROM_SQL_STRING = 1, + DATA_FROM_DATA_FILE = 2, +}; typedef SCMSTableVgroupRspMsg SVgroupsInfo; @@ -47,20 +55,23 @@ typedef struct STableComInfo { } STableComInfo; typedef struct STableMeta { - //super table if it is created according to super table, otherwise, tableInfo is used - union { struct STableMeta* pSTable; STableComInfo tableInfo; }; - uint8_t tableType; - int16_t sversion; + // super table if it is created according to super table, otherwise, tableInfo is used + union { + struct STableMeta *pSTable; + STableComInfo tableInfo; + }; + uint8_t tableType; + int16_t sversion; SCMVgroupInfo vgroupInfo; - int32_t sid; // the index of one table in a virtual node - uint64_t uid; // unique id of a table - SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info + int32_t sid; // the index of one table in a virtual node + uint64_t uid; // unique id of a table + SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; typedef struct STableMetaInfo { - STableMeta * pTableMeta; // table meta, cached in client side and acquried by name - SVgroupsInfo* vgroupList; - + STableMeta * pTableMeta; // table meta, cached in client side and acquried by name + SVgroupsInfo *vgroupList; + /* * 1. keep the vnode index during the multi-vnode super table projection query * 2. keep the vnode index for multi-vnode insertion @@ -73,16 +84,16 @@ typedef struct STableMetaInfo { /* the structure for sql function in select clause */ typedef struct SSqlExpr { - char aliasName[TSDB_COL_NAME_LEN]; // as aliasName - SColIndex colInfo; - int64_t uid; // refactor use the pointer - int16_t functionId; // function id in aAgg array - int16_t resType; // return value type - int16_t resBytes; // length of return value - int16_t interResBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - tVariant param[3]; // parameters are not more than 3 - int32_t offset; // sub result column value of arithmetic expression. + char aliasName[TSDB_COL_NAME_LEN]; // as aliasName + SColIndex colInfo; + int64_t uid; // refactor use the pointer + int16_t functionId; // function id in aAgg array + int16_t resType; // return value type + int16_t resBytes; // length of return value + int16_t interResBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + tVariant param[3]; // parameters are not more than 3 + int32_t offset; // sub result column value of arithmetic expression. } SSqlExpr; typedef struct SColumnIndex { @@ -92,14 +103,14 @@ typedef struct SColumnIndex { typedef struct SFieldSupInfo { bool visible; - SArithExprInfo* pArithExprInfo; - SSqlExpr* pSqlExpr; + SArithExprInfo *pArithExprInfo; + SSqlExpr * pSqlExpr; } SFieldSupInfo; typedef struct SFieldInfo { - int16_t numOfOutput; // number of column in result - SArray* pFields; // SArray - SArray* pSupportInfo; // SArray + int16_t numOfOutput; // number of column in result + SArray *pFields; // SArray + SArray *pSupportInfo; // SArray } SFieldInfo; typedef struct SColumn { @@ -108,11 +119,9 @@ typedef struct SColumn { SColumnFilterInfo *filterInfo; } SColumn; -struct SLocalReducer; - typedef struct SCond { uint64_t uid; - int32_t len; // length of tag query condition data + int32_t len; // length of tag query condition data char * cond; } SCond; @@ -139,7 +148,7 @@ typedef struct STagCond { SJoinInfo joinInfo; // for different table, the query condition must be seperated - SArray* pCond; + SArray *pCond; } STagCond; typedef struct SParamInfo { @@ -180,7 +189,7 @@ typedef struct STableDataBlocks { SParamInfo *params; } STableDataBlocks; -typedef struct SDataBlockList { // todo remove +typedef struct SDataBlockList { // todo remove uint32_t nSize; uint32_t nAlloc; STableDataBlocks **pData; @@ -196,9 +205,9 @@ typedef struct SQueryInfo { int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info - SArray* colList; // SArray + SArray * colList; // SArray SFieldInfo fieldsInfo; - SArray* exprsInfo; // SArray + SArray * exprsInfo; // SArray SLimitVal limit; SLimitVal slimit; STagCond tagCond; @@ -215,19 +224,13 @@ typedef struct SQueryInfo { int64_t prjOffset; } SQueryInfo; -// data source from sql string or from file -enum { - DATA_FROM_SQL_STRING = 1, - DATA_FROM_DATA_FILE = 2, -}; - typedef struct { int command; uint8_t msgType; union { bool existsCheck; // check if the table exists or not - bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta + bool autoCreated; // if the table is missing, on-the-fly create it. during getmeterMeta int8_t dataSourceType; // load data from file or not }; @@ -244,11 +247,11 @@ typedef struct { int32_t payloadLen; SQueryInfo **pQueryInfo; int32_t numOfClause; - - SDataBlockList *pDataBlocks; // submit data blocks after parsing sql - char * curSql; // current sql, resume position of sql after parsing paused - void * pTableList; // referred table involved in sql - + + SDataBlockList *pDataBlocks; // submit data blocks after parsing sql + char * curSql; // current sql, resume position of sql after parsing paused + void * pTableList; // referred table involved in sql + // for parameter ('?') binding and batch processing int32_t batchSize; int32_t numOfParams; @@ -259,50 +262,48 @@ typedef struct SResRec { int numOfTotal; } SResRec; -struct STSBuf; - typedef struct { - int64_t numOfRows; // num of results in current retrieved - int64_t numOfTotal; // num of total results - int64_t numOfTotalInCurrentClause; // num of total result in current subclause - char * pRsp; - int32_t rspType; - int32_t rspLen; - uint64_t qhandle; - int64_t uid; - int64_t useconds; - int64_t offset; // offset value from vnode during projection query of stable - int32_t row; - int16_t numOfCols; - int16_t precision; - bool completed; - int32_t code; - int32_t numOfGroups; - SResRec * pGroupRec; - char * data; - void ** tsrow; - char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) - SColumnIndex *pColumnIndex; + int64_t numOfRows; // num of results in current retrieved + int64_t numOfTotal; // num of total results + int64_t numOfTotalInCurrentClause; // num of total result in current subclause + char * pRsp; + int32_t rspType; + int32_t rspLen; + uint64_t qhandle; + int64_t uid; + int64_t useconds; + int64_t offset; // offset value from vnode during projection query of stable + int32_t row; + int16_t numOfCols; + int16_t precision; + bool completed; + int32_t code; + int32_t numOfGroups; + SResRec * pGroupRec; + char * data; + void ** tsrow; + char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) + SColumnIndex * pColumnIndex; struct SLocalReducer *pLocalReducer; } SSqlRes; typedef struct STscObj { - void * signature; - void * pTimer; - char mgmtIp[TSDB_USER_LEN]; - uint16_t mgmtPort; - char user[TSDB_USER_LEN]; - char pass[TSDB_KEY_LEN]; - char acctId[TSDB_DB_NAME_LEN]; - char db[TSDB_TABLE_ID_LEN]; - char sversion[TSDB_VERSION_LEN]; - char writeAuth : 1; - char superAuth : 1; - struct SSqlObj *pSql; - struct SSqlObj *pHb; - struct SSqlObj *sqlList; + void * signature; + void * pTimer; + char mgmtIp[TSDB_USER_LEN]; + uint16_t mgmtPort; + char user[TSDB_USER_LEN]; + char pass[TSDB_KEY_LEN]; + char acctId[TSDB_DB_NAME_LEN]; + char db[TSDB_TABLE_ID_LEN]; + char sversion[TSDB_VERSION_LEN]; + char writeAuth : 1; + char superAuth : 1; + struct SSqlObj * pSql; + struct SSqlObj * pHb; + struct SSqlObj * sqlList; struct SSqlStream *streamList; - pthread_mutex_t mutex; + pthread_mutex_t mutex; } STscObj; typedef struct SSqlObj { @@ -310,23 +311,23 @@ typedef struct SSqlObj { STscObj *pTscObj; void (*fp)(); void (*fetchFp)(); - void * param; - uint32_t ip; - short vnode; - int64_t stime; - uint32_t queryId; - void * pStream; - void * pSubscription; - char * sqlstr; - char retry; - char maxRetry; - SRpcIpSet ipList; - char freed : 4; - char listed : 4; - tsem_t rspSem; - SSqlCmd cmd; - SSqlRes res; - uint8_t numOfSubs; + void * param; + uint32_t ip; + short vnode; + int64_t stime; + uint32_t queryId; + void * pStream; + void * pSubscription; + char * sqlstr; + char retry; + char maxRetry; + SRpcIpSet ipList; + char freed : 4; + char listed : 4; + tsem_t rspSem; + SSqlCmd cmd; + SSqlRes res; + uint8_t numOfSubs; struct SSqlObj **pSubs; struct SSqlObj * prev, *next; } SSqlObj; @@ -360,13 +361,10 @@ typedef struct SSqlStream { } SSqlStream; int32_t tscInitRpc(const char *user, const char *secret); +void tscInitMsgsFp(); -// tscSql API int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion); -void tscInitMsgsFp(); -extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); - void tscProcessMsgFromServer(SRpcMsg *rpcMsg); int tscProcessSql(SSqlObj *pSql); @@ -379,12 +377,8 @@ int tscProcessLocalCmd(SSqlObj *pSql); int tscCfgDynamicOptions(char *msg); int taos_retrieve(TAOS_RES *res); -/* - * transfer function for metric query in stream computing, the function need to be change - * before send query message to vnode - */ -int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo *pQueryInfo); -void tscRestoreSQLFunctionForMetricQuery(SQueryInfo *pQueryInfo); +int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); +void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscDestroyResPointerInfo(SSqlRes *pRes); @@ -414,10 +408,13 @@ void tscFreeSqlObj(SSqlObj *pObj); void tscCloseTscObj(STscObj *pObj); -void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen); +TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), + void *param, void **taos); + +void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen); void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql); -void tscKillMetricQuery(SSqlObj *pSql); +void tscKillSTableQuery(SSqlObj *pSql); void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen); bool tscIsUpdateQuery(STscObj *pObj); bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); @@ -426,20 +423,22 @@ char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); -void tscQueueAsyncFreeResult(SSqlObj *pSql); -int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo); -char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column); +void tscQueueAsyncFreeResult(SSqlObj *pSql); +int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); +char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column); -extern void * pVnodeConn; -extern void * pTscMgmtConn; -extern void * tscCacheHandle; -extern int slaveIndex; -extern void * tscTmr; -extern void * tscQhandle; -extern int tscKeepConn[]; -extern int tsInsertHeadSize; -extern int tscNumOfThreads; -extern SRpcIpSet tscMgmtIpSet; +extern void * pVnodeConn; +extern void * pTscMgmtConn; +extern void * tscCacheHandle; +extern int slaveIndex; +extern void * tscTmr; +extern void * tscQhandle; +extern int tscKeepConn[]; +extern int tsInsertHeadSize; +extern int tscNumOfThreads; +extern SRpcIpSet tscMgmtIpSet; + +extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 7357da5d17..99d20de48b 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -488,7 +488,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { */ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - tscTansformSQLFunctionForSTableQuery(pQueryInfo); + tscTansformSQLFuncForSTableQuery(pQueryInfo); tscIncStreamExecutionCount(pSql->pStream); } else { tscTrace("%p get tableMeta successfully", pSql); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 6482b00654..8272de7d16 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -296,7 +296,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { int32_t rowLen = tscBuildMeterSchemaResultFields(pSql, NUM_OF_DESCRIBE_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, note_field_length); - tscFieldInfoCalOffset(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); return tscSetValueToResObj(pSql, rowLen); } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 254ad1cb9f..cab7e15023 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -613,7 +613,7 @@ static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, in } // data block is disordered, sort it in ascending order -void sortRemoveDuplicates(STableDataBlocks *dataBuf) { +void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; // size is less than the total size, since duplicated rows may be removed yet. diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c7be16c457..8b7cea356f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1258,7 +1258,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel * transfer sql functions that need secondary merge into another format * in dealing with metric queries such as: count/first/last */ - tscTansformSQLFunctionForSTableQuery(pQueryInfo); + tscTansformSQLFuncForSTableQuery(pQueryInfo); if (hasUnsupportFunctionsForSTableQuery(pQueryInfo)) { return TSDB_CODE_INVALID_SQL; @@ -2242,7 +2242,7 @@ bool validateIpAddress(const char* ip, size_t size) { return ipAddr != INADDR_NONE; } -int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { +int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { @@ -2282,7 +2282,7 @@ int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo* pQueryInfo) { } /* transfer the field-info back to original input format */ -void tscRestoreSQLFunctionForMetricQuery(SQueryInfo* pQueryInfo) { +void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return; @@ -2547,7 +2547,7 @@ void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo) { if (QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { tscFieldInfoUpdateOffsetForInterResult(pQueryInfo); } else { - tscFieldInfoCalOffset(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); } } @@ -2828,7 +2828,7 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn const char* msg1 = "non binary column not support like operator"; const char* msg2 = "binary column not support this operator"; - SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex); + SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex); SColumnFilterInfo* pColFilter = NULL; /* diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 8b1ea1f328..7ad3c08034 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -168,7 +168,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->sid = pTableMetaMsg->sid; pTableMeta->uid = pTableMetaMsg->uid; - pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; +// pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index 6631add297..bd2e6e9019 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -261,8 +261,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->pCtx = (SQLFunctionCtx *)calloc(size, sizeof(SQLFunctionCtx)); pReducer->rowSize = pMemBuffer[0]->nElemSize; - tscRestoreSQLFunctionForMetricQuery(pQueryInfo); - tscFieldInfoCalOffset(pQueryInfo); + tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); if (pReducer->rowSize > pMemBuffer[0]->pageSize) { assert(false); // todo fixed row size is larger than the minimum page size; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ec4d2a927f..332d56c1b1 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -458,7 +458,7 @@ int tscProcessSql(SSqlObj *pSql) { return doProcessSql(pSql); } -void tscKillMetricQuery(SSqlObj *pSql) { +void tscKillSTableQuery(SSqlObj *pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -2202,7 +2202,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes); } - tscFieldInfoCalOffset(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); tfree(pTableMeta); return 0; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 6855ea8295..f70beab166 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -964,7 +964,7 @@ void taos_stop_query(TAOS_RES *res) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - tscKillMetricQuery(pSql); + tscKillSTableQuery(pSql); return; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 4802bb293a..bae1c0f86a 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -86,7 +86,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; } - tscTansformSQLFunctionForSTableQuery(pQueryInfo); + tscTansformSQLFuncForSTableQuery(pQueryInfo); // failed to get meter/metric meta, retry in 10sec. if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 46e859c221..00a4c8c448 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -305,7 +305,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); pQueryInfo->exprsInfo = tscSqlExprCopy(pSupporter->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopyAll(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); + tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); pSupporter->fieldsInfo.numOfOutput = 0; @@ -321,7 +321,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - tscFieldInfoCalOffset(pNewQueryInfo); + tscFieldInfoUpdateOffset(pNewQueryInfo); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); @@ -859,7 +859,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu tscColumnListAssign(pSupporter->colList, pNewQueryInfo->colList, 0); pSupporter->exprsInfo = tscSqlExprCopy(pNewQueryInfo->exprsInfo, pSupporter->uid, false); - tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); + tscFieldInfoCopy(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7de2f0ce70..d0fbc4c373 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -475,6 +475,7 @@ SDataBlockList* tscCreateBlockArrayList() { if (pDataBlockArrayList == NULL) { return NULL; } + pDataBlockArrayList->nAlloc = DEFAULT_INITIAL_NUM_OF_BLOCK; pDataBlockArrayList->pData = calloc(1, POINTER_BYTES * pDataBlockArrayList->nAlloc); if (pDataBlockArrayList->pData == NULL) { @@ -716,7 +717,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi } SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData; - sortRemoveDuplicates(pOneTableBlock); + tscSortRemoveDataBlockDupRows(pOneTableBlock); char* e = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); @@ -838,7 +839,7 @@ SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FI return taosArrayInsert(pFieldInfo->pSupportInfo, index, &info); } -void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo) { +void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprsInfo, 0); @@ -869,26 +870,7 @@ void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) { } } -void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList, int32_t size) { - if (src == NULL) { - return; - } - - if (size <= 0) { - tscFieldInfoCopyAll(dst, src); - } else { // only copy the required column - for (int32_t i = 0; i < size; ++i) { - assert(indexList[i] >= 0 && indexList[i] <= src->numOfOutput); - TAOS_FIELD* p = taosArrayGet(src->pFields, indexList[i]); - SFieldSupInfo* pInfo = taosArrayGet(src->pSupportInfo, indexList[i]); - - SFieldSupInfo* pInfo1 = tscFieldInfoAppend(dst, p); - *pInfo1 = *pInfo; - } - } -} - -void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src) { +void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) { dst->numOfOutput = src->numOfOutput; taosArrayCopy(dst->pFields, src->pFields); @@ -1818,20 +1800,22 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo); - if (numOfOutput > 0) { - int32_t* indexList = calloc(1, numOfOutput * sizeof(int32_t)); + if (numOfOutput > 0) { // todo refactor to extract method size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - - for (int32_t i = 0, j = 0; i < numOfExprs; ++i) { + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->uid == uid) { - indexList[j++] = i; + TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); + *pInfo1 = *pInfo; } } - tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pNewQueryInfo->fieldsInfo, indexList, numOfOutput); - free(indexList); - // make sure the the sqlExpr for each fields is correct // todo handle the agg arithmetic expression for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { @@ -1930,9 +1914,7 @@ bool tscIsUpdateQuery(STscObj* pObj) { SSqlCmd* pCmd = &pObj->pSql->cmd; return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || - TSDB_SQL_USE_DB == pCmd->command) - ? 1 - : 0; + TSDB_SQL_USE_DB == pCmd->command); } int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) { diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index caaae9e988..3b5784986e 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -364,7 +364,7 @@ static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCta bool find = false; int32_t pos = 0; - for (int pos = 0; pos < pStable->vgLen; ++pos) { + for (pos = 0; pos < pStable->vgLen; ++pos) { if (pStable->vgList[pos] == 0) break; if (pStable->vgList[pos] == pCtable->vgId) { find = true; @@ -1134,7 +1134,7 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { static int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags; for (int32_t i = 0; i < numOfCols; ++i) { - strcpy(pSchema->name, pTable->schema[i].name); + strncpy(pSchema->name, pTable->schema[i].name, TSDB_TABLE_ID_LEN); pSchema->type = pTable->schema[i].type; pSchema->bytes = htons(pTable->schema[i].bytes); pSchema->colId = htons(pTable->schema[i].colId); @@ -1154,7 +1154,7 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) { pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->tableType = pTable->info.type; pMeta->contLen = sizeof(STableMetaMsg) + mgmtSetSchemaFromSuperTable(pMeta->schema, pTable); - strcpy(pMeta->tableId, pTable->info.tableId); + strncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN); SRpcMsg rpcRsp = { .handle = pMsg->thandle, From e424202e578c8e19114cdd3035920fe2d0798d09 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 23 Apr 2020 15:06:20 +0800 Subject: [PATCH 07/63] [td-171] refactor codes --- src/client/inc/tsclient.h | 6 ++-- src/client/src/tscSQLParser.c | 55 ++++++++++++++---------------- src/client/src/tscSecondaryMerge.c | 10 +++--- src/client/src/tscServer.c | 8 ++--- src/client/src/tscStream.c | 20 +++++------ src/client/src/tscSubquery.c | 6 ++-- src/client/src/tscUtil.c | 4 +-- 7 files changed, 53 insertions(+), 56 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 89bed123c4..337b6b9628 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -200,14 +200,14 @@ typedef struct SQueryInfo { uint16_t type; // query/insert/import type char slidingTimeUnit; - int64_t etime, stime; + STimeWindow window; int64_t intervalTime; // aggregation time interval int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info - SArray * colList; // SArray + SArray * colList; // SArray SFieldInfo fieldsInfo; - SArray * exprsInfo; // SArray + SArray * exprsInfo; // SArray SLimitVal limit; SLimitVal slimit; STagCond tagCond; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 8b7cea356f..fce83c8790 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2656,7 +2656,7 @@ typedef struct SCondExpr { bool tsJoin; } SCondExpr; -static int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t optr, int16_t timePrecision); +static int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision); static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) { if (pExpr->nSQLOptr == TK_ID) { // column name @@ -3631,20 +3631,18 @@ static int32_t getTimeRangeFromExpr(SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { tSQLExpr* pRight = pExpr->pRight; - TSKEY stime = 0; - TSKEY etime = INT64_MAX; - - if (getTimeRange(&stime, &etime, pRight, pExpr->nSQLOptr, tinfo.precision) != TSDB_CODE_SUCCESS) { + STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; + if (getTimeRange(&win, pRight, pExpr->nSQLOptr, tinfo.precision) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(pQueryInfo->msg, msg0); } // update the timestamp query range - if (pQueryInfo->stime < stime) { - pQueryInfo->stime = stime; + if (pQueryInfo->window.skey < win.skey) { + pQueryInfo->window.skey = win.skey; } - if (pQueryInfo->etime > etime) { - pQueryInfo->etime = etime; + if (pQueryInfo->window.ekey > win.ekey) { + pQueryInfo->window.ekey = win.ekey; } } @@ -3756,8 +3754,8 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql int32_t ret = TSDB_CODE_SUCCESS; - pQueryInfo->stime = 0; - pQueryInfo->etime = INT64_MAX; + pQueryInfo->window.skey = 0; + pQueryInfo->window.ekey = INT64_MAX; // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space SStringBuilder sb = {0}; @@ -3823,7 +3821,7 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql return ret; } -int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t optr, int16_t timePrecision) { +int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision) { // this is join condition, do nothing if (pRight->nSQLOptr == TK_ID) { return TSDB_CODE_SUCCESS; @@ -3898,16 +3896,15 @@ int32_t getTimeRange(int64_t* stime, int64_t* etime, tSQLExpr* pRight, int32_t o } if (optr == TK_LE) { - *etime = val; + win->ekey = val; } else if (optr == TK_LT) { - *etime = val - delta; + win->ekey = val - delta; } else if (optr == TK_GT) { - *stime = val + delta; + win->skey = val + delta; } else if (optr == TK_GE) { - *stime = val; + win->skey = val; } else if (optr == TK_EQ) { - *stime = val; - *etime = *stime; + win->ekey = win->skey = val; } return TSDB_CODE_SUCCESS; } @@ -4612,7 +4609,7 @@ bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo) { return true; } - return (pQueryInfo->stime == pQueryInfo->etime) && (pQueryInfo->stime != 0); + return (pQueryInfo->window.skey == pQueryInfo->window.ekey) && (pQueryInfo->window.skey != 0); } int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* pQuerySql, SSqlObj* pSql) { @@ -5757,22 +5754,22 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { pQuerySql->pWhere = NULL; if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { - pQueryInfo->stime = pQueryInfo->stime / 1000; - pQueryInfo->etime = pQueryInfo->etime / 1000; + pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; + pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; } } else { // set the time rang - pQueryInfo->stime = 0; - pQueryInfo->etime = INT64_MAX; + pQueryInfo->window.skey = 0; + pQueryInfo->window.ekey = INT64_MAX; } // user does not specified the query time window, twa is not allowed in such case. - if ((pQueryInfo->stime == 0 || pQueryInfo->etime == INT64_MAX || - (pQueryInfo->etime == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { + if ((pQueryInfo->window.skey == 0 || pQueryInfo->window.ekey == INT64_MAX || + (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); } // no result due to invalid query time range - if (pQueryInfo->stime > pQueryInfo->etime) { + if (pQueryInfo->window.skey > pQueryInfo->window.ekey) { pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; return TSDB_CODE_SUCCESS; } @@ -5783,9 +5780,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { // in case of join query, time range is required. if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { - int64_t timeRange = labs(pQueryInfo->stime - pQueryInfo->etime); + int64_t timeRange = labs(pQueryInfo->window.skey - pQueryInfo->window.ekey); - if (timeRange == 0 && pQueryInfo->stime == 0) { + if (timeRange == 0 && pQueryInfo->window.skey == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } } @@ -5814,7 +5811,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } if (pQueryInfo->intervalTime > 0) { - int64_t timeRange = labs(pQueryInfo->stime - pQueryInfo->etime); + int64_t timeRange = labs(pQueryInfo->window.skey - pQueryInfo->window.ekey); // number of result is not greater than 10,000,000 if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY) { return invalidSqlErrMsg(pQueryInfo->msg, msg6); diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index bd2e6e9019..84f14abf4c 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -327,7 +327,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); int16_t prec = tinfo.precision; - int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; + int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); @@ -787,7 +787,7 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); int16_t prec = tinfo.precision; - int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; + int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, prec); @@ -912,7 +912,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; - int64_t actualETime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; + int64_t actualETime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { @@ -1295,7 +1295,7 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer // for group result interpolation, do not return if not data is generated if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; + int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t newTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); @@ -1361,7 +1361,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { prevGroupCompleted) { // if interpoType == TSDB_INTERPO_NONE, return directly if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { - int64_t etime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; + int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 332d56c1b1..88d0be3fb9 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -669,11 +669,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - pQueryMsg->window.skey = htobe64(pQueryInfo->stime); - pQueryMsg->window.ekey = htobe64(pQueryInfo->etime); + pQueryMsg->window.skey = htobe64(pQueryInfo->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryInfo->window.ekey); } else { - pQueryMsg->window.skey = htobe64(pQueryInfo->etime); - pQueryMsg->window.ekey = htobe64(pQueryInfo->stime); + pQueryMsg->window.skey = htobe64(pQueryInfo->window.ekey); + pQueryMsg->window.ekey = htobe64(pQueryInfo->window.skey); } pQueryMsg->numOfTables = htonl(numOfTables); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index bae1c0f86a..9f7d4887d1 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -116,18 +116,18 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { if (isProjectStream(pQueryInfo)) { /* - * pQueryInfo->etime, which is the start time, does not change in case of + * pQueryInfo->window.ekey, which is the start time, does not change in case of * repeat first execution, once the first execution failed. */ - pQueryInfo->stime = pStream->stime; // start time + pQueryInfo->window.skey = pStream->stime; // start time - pQueryInfo->etime = taosGetTimestamp(pStream->precision); // end time - if (pQueryInfo->etime > pStream->etime) { - pQueryInfo->etime = pStream->etime; + pQueryInfo->window.ekey = taosGetTimestamp(pStream->precision); // end time + if (pQueryInfo->window.ekey > pStream->etime) { + pQueryInfo->window.ekey = pStream->etime; } } else { - pQueryInfo->stime = pStream->stime - pStream->interval; - pQueryInfo->etime = pStream->stime - 1; + pQueryInfo->window.skey = pStream->stime - pStream->interval; + pQueryInfo->window.ekey = pStream->stime - 1; } // launch stream computing in a new thread @@ -425,10 +425,10 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in pStream->slidingTime = tsProjectExecInterval; if (stime != 0) { // first projection start from the latest event timestamp - assert(stime >= pQueryInfo->stime); + assert(stime >= pQueryInfo->window.skey); stime += 1; // exclude the last records from table } else { - stime = pQueryInfo->stime; + stime = pQueryInfo->window.skey; } } else { // timewindow based aggregation stream if (stime == 0) { // no data in meter till now @@ -548,7 +548,7 @@ TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *p pStream->precision = tinfo.precision; pStream->ctime = taosGetTimestamp(pStream->precision); - pStream->etime = pQueryInfo->etime; + pStream->etime = pQueryInfo->window.ekey; pSql->pStream = pStream; tscAddIntoStreamList(pStream); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 00a4c8c448..fba82657aa 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -416,10 +416,10 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSubquerySupporter* pSupporter // update the query time range according to the join results on timestamp static void updateQueryTimeRange(SQueryInfo* pQueryInfo, int64_t st, int64_t et) { - assert(pQueryInfo->stime <= st && pQueryInfo->etime >= et); + assert(pQueryInfo->window.skey <= st && pQueryInfo->window.ekey >= et); - pQueryInfo->stime = st; - pQueryInfo->etime = et; + pQueryInfo->window.skey = st; + pQueryInfo->window.ekey = et; } static void joinRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d0fbc4c373..d7db533e7d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1866,8 +1866,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void "%p new subquery: %p, tableIndex:%d, vnodeIdx:%d, type:%d, exprInfo:%d, colList:%d," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), - size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->stime, - pNewQueryInfo->etime, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); + size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->window.skey, + pNewQueryInfo->window.ekey, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); tscPrintSelectClause(pNew, 0); } else { From 7293ebe830543d37f0c9112776dd496862b2e62a Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 23 Apr 2020 16:02:51 +0800 Subject: [PATCH 08/63] [TD-192] --- packaging/tools/makepkg.sh | 2 +- src/CMakeLists.txt | 2 +- src/kit/CMakeLists.txt | 4 ++-- src/kit/taosdump/CMakeLists.txt | 1 + src/kit/taosdump/taosdump.c | 8 ++++---- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index d39cf41843..c214626efd 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -43,7 +43,7 @@ lib_files="${build_dir}/lib/libtaos.so.${version}" header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" cfg_dir="${top_dir}/packaging/cfg" install_files="${script_dir}/install.sh" -nginx_dir="${code_dir}/../../enterprise/src/modules/web" +nginx_dir="${code_dir}/../../enterprise/src/plugins/web" # Init file #init_dir=${script_dir}/deb diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a8f7d8807..904050fb1b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,4 +15,4 @@ ADD_SUBDIRECTORY(vnode) ADD_SUBDIRECTORY(tsdb) ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(dnode) -#ADD_SUBDIRECTORY(connector/jdbc) +ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 386c8a92f7..66e8cf7398 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -2,5 +2,5 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) -#ADD_SUBDIRECTORY(taosdemo) -#ADD_SUBDIRECTORY(taosdump) +ADD_SUBDIRECTORY(taosdemo) +ADD_SUBDIRECTORY(taosdump) diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index a15b0ff552..c63168237c 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -2,6 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 4b613aefdc..bb3aa46fb1 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -30,7 +30,7 @@ #include "taos.h" #include "taosmsg.h" -#include "tglobal.h" +//#include "tglobal.h" #include "tsclient.h" #include "taosdef.h" #include "tutil.h" @@ -360,7 +360,7 @@ int main(int argc, char *argv[]) { void taosFreeDbInfos() { if (dbInfos == NULL) return; - for (int i = 0; i < tsMaxDbs; i++) tfree(dbInfos[i]); + for (int i = 0; i < TSDB_MAX_DBS; i++) tfree(dbInfos[i]); tfree(dbInfos); } @@ -440,7 +440,7 @@ int taosDumpOut(struct arguments *arguments) { return -1; } - dbInfos = (SDbInfo **)calloc(tsMaxDbs, sizeof(SDbInfo *)); + dbInfos = (SDbInfo **)calloc(TSDB_MAX_DBS, sizeof(SDbInfo *)); if (dbInfos == NULL) { fprintf(stderr, "failed to allocate memory\n"); goto _exit_failure; @@ -1330,4 +1330,4 @@ _exit_no_charset: *fcharset = '\0'; tfree(line); return; -} \ No newline at end of file +} From 5f58f971d0dfee0047c996545f87422aeb29cab0 Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 23 Apr 2020 16:05:34 +0800 Subject: [PATCH 09/63] [TD-192] --- src/kit/taosdump/taosdump.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index bb3aa46fb1..2e64c9bccc 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -30,7 +30,6 @@ #include "taos.h" #include "taosmsg.h" -//#include "tglobal.h" #include "tsclient.h" #include "taosdef.h" #include "tutil.h" From 6a851fde8e5d03ca4b1acd23b536f0d6a09da930 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 23 Apr 2020 16:42:48 +0800 Subject: [PATCH 10/63] [TD-193] authority in account manage --- src/mnode/src/mgmtDb.c | 4 ++- src/mnode/src/mgmtDnode.c | 6 ++-- src/mnode/src/mgmtSdb.c | 6 ++-- src/mnode/src/mgmtTable.c | 36 +++++++++---------- src/mnode/src/mgmtUser.c | 4 +++ src/mnode/src/mgmtVgroup.c | 13 ++++--- tests/script/sh/deploy.sh | 2 +- .../script/unique/account/account_delete.sim | 13 +++++-- tests/script/unique/account/basic.sim | 2 +- tests/script/unique/account/monitor.sim | 36 ------------------- tests/script/unique/account/paras.sim | 2 +- tests/script/unique/account/testSuite.sim | 3 ++ tests/script/unique/account/usage.sim | 2 +- 13 files changed, 57 insertions(+), 72 deletions(-) delete mode 100644 tests/script/unique/account/monitor.sim diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index d57b75501a..a1b5f12ffb 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -884,6 +884,8 @@ void mgmtDropAllDbs(SAcctObj *pAcct) { SDbObj *pDb = NULL; void * pNode = NULL; + mPrint("acct:%s, all dbs will be dropped from sdb", pAcct->user); + while (1) { pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb); if (pDb == NULL) break; @@ -902,5 +904,5 @@ void mgmtDropAllDbs(SAcctObj *pAcct) { mgmtDecDbRef(pDb); } - mTrace("acct:%s, all dbs is is dropped from sdb", pAcct->user, numOfDbs); + mPrint("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 562e6c0589..8afeadeb0c 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -260,7 +260,7 @@ void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) { } uint32_t dnodeIp = inet_addr(pCmCfgDnode->ip); - if (strcmp(pMsg->pUser->pAcct->user, "root") != 0) { + if (strcmp(pMsg->pUser->user, "root") != 0) { rpcRsp.code = TSDB_CODE_NO_RIGHTS; } else { SRpcIpSet ipSet = mgmtGetIpSetFromIp(dnodeIp); @@ -469,7 +469,7 @@ static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg) { SCMCreateDnodeMsg *pCreate = pMsg->pCont; - if (strcmp(pMsg->pUser->pAcct->user, "root") != 0) { + if (strcmp(pMsg->pUser->user, "root") != 0) { rpcRsp.code = TSDB_CODE_NO_RIGHTS; } else { uint32_t ip = inet_addr(pCreate->ip); @@ -489,7 +489,7 @@ static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg) { SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0}; SCMDropDnodeMsg *pDrop = pMsg->pCont; - if (strcmp(pMsg->pUser->pAcct->user, "root") != 0) { + if (strcmp(pMsg->pUser->user, "root") != 0) { rpcRsp.code = TSDB_CODE_NO_RIGHTS; } else { uint32_t ip = inet_addr(pDrop->ip); diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 9e4bfb453b..0aae756529 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -333,7 +333,7 @@ void sdbIncRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); atomic_add_fetch_32(pRefCount, 1); - if (0 && strcmp(pTable->tableName, "accounts") == 0) { + if (1 && strcmp(pTable->tableName, "accounts") == 0) { sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } @@ -345,7 +345,7 @@ void sdbDecRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - if (0 && strcmp(pTable->tableName, "accounts") == 0) { + if (1 && strcmp(pTable->tableName, "accounts") == 0) { sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } @@ -400,6 +400,8 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { if (pTable->keyType == SDB_KEY_AUTO) { pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pOper->pObj)); + } else { + pTable->autoIndex++; } pthread_mutex_unlock(&pTable->mutex); diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 3b14377482..b8228e8e47 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -1109,12 +1109,11 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { int32_t dbNameLen = strlen(pDropDb->name); SSuperTableObj *pTable = NULL; + mPrint("db:%s, all super tables will be dropped from sdb", pDropDb->name); + while (1) { - mgmtDecTableRef(pTable); pNode = sdbFetchRow(tsSuperTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) { - break; - } + if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOper oper = { @@ -1125,10 +1124,12 @@ void mgmtDropAllSuperTables(SDbObj *pDropDb) { sdbDeleteRow(&oper); pNode = pLastNode; numOfTables ++; - continue; } + + mgmtDecTableRef(pTable); } - mTrace("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); + + mPrint("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } static int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { @@ -1678,12 +1679,11 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { int32_t dbNameLen = strlen(pDropDb->name); SChildTableObj *pTable = NULL; + mPrint("db:%s, all child tables will be dropped from sdb", pDropDb->name); + while (1) { - mgmtDecTableRef(pTable); pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) { - break; - } + if (pTable == NULL) break; if (strncmp(pDropDb->name, pTable->info.tableId, dbNameLen) == 0) { SSdbOper oper = { @@ -1694,11 +1694,11 @@ void mgmtDropAllChildTables(SDbObj *pDropDb) { sdbDeleteRow(&oper); pNode = pLastNode; numOfTables++; - continue; } + mgmtDecTableRef(pTable); } - mTrace("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); + mPrint("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); } static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { @@ -1707,12 +1707,11 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { int32_t numOfTables = 0; SChildTableObj *pTable = NULL; + mPrint("stable:%s, all child tables will dropped from sdb", pStable->info.tableId, numOfTables); + while (1) { - mgmtDecTableRef(pTable); pNode = sdbFetchRow(tsChildTableSdb, pNode, (void **)&pTable); - if (pTable == NULL) { - break; - } + if (pTable == NULL) break; if (pTable->superTable == pStable) { SSdbOper oper = { @@ -1723,11 +1722,12 @@ static void mgmtDropAllChildTablesInStable(SSuperTableObj *pStable) { sdbDeleteRow(&oper); pNode = pLastNode; numOfTables++; - continue; } + + mgmtDecTableRef(pTable); } - mTrace("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); + mPrint("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); } static SChildTableObj* mgmtGetTableByPos(uint32_t dnodeId, int32_t vnode, int32_t sid) { diff --git a/src/mnode/src/mgmtUser.c b/src/mnode/src/mgmtUser.c index a2ec2a32ad..708f436d0a 100644 --- a/src/mnode/src/mgmtUser.c +++ b/src/mnode/src/mgmtUser.c @@ -353,6 +353,7 @@ static void mgmtProcessCreateUserMsg(SQueuedMsg *pMsg) { mLPrint("user:%s, is created by %s", pCreate->user, pOperUser->user); } } else { + mError("user:%s, no rights to create user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } @@ -398,6 +399,7 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { code = mgmtUpdateUser(pUser); mLPrint("user:%s, password is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); } else { + mError("user:%s, no rights to ater user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } @@ -440,11 +442,13 @@ static void mgmtProcessAlterUserMsg(SQueuedMsg *pMsg) { code = mgmtUpdateUser(pUser); mLPrint("user:%s, privilege is altered by %s, result:%s", pUser->user, pOperUser->user, tstrerror(code)); } else { + mError("user:%s, no rights to ater user", pOperUser->user); code = TSDB_CODE_NO_RIGHTS; } mgmtSendSimpleResp(pMsg->thandle, code); } else { + mError("user:%s, no rights to ater user", pOperUser->user); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_NO_RIGHTS); } diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index 4088b37e8a..bec4dbe83a 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -744,12 +744,13 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) { void mgmtDropAllVgroups(SDbObj *pDropDb) { void *pNode = NULL; void *pLastNode = NULL; - int32_t numOfTables = 0; + int32_t numOfVgroups = 0; int32_t dbNameLen = strlen(pDropDb->name); SVgObj *pVgroup = NULL; + mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name); + while (1) { - mgmtDecVgroupRef(pVgroup); pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup); if (pVgroup == NULL) break; @@ -761,12 +762,14 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) { }; sdbDeleteRow(&oper); pNode = pLastNode; - numOfTables++; - continue; + numOfVgroups++; } + + mgmtSendDropVgroupMsg(pVgroup, NULL); + mgmtDecVgroupRef(pVgroup); } - mTrace("db:%s, all vgroups is dropped from sdb", pDropDb->name, numOfTables); + mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups); } void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 41ba3c425a..b1aa7c6382 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -93,7 +93,7 @@ echo "privateIp $NODE_IP" >> $TAOS_CFG echo "dDebugFlag 199" >> $TAOS_CFG echo "mDebugFlag 199" >> $TAOS_CFG echo "sdbDebugFlag 199" >> $TAOS_CFG -echo "rpcDebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 131" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "httpDebugFlag 131" >> $TAOS_CFG diff --git a/tests/script/unique/account/account_delete.sim b/tests/script/unique/account/account_delete.sim index 35cdcabfc7..f4bc8d0a2f 100644 --- a/tests/script/unique/account/account_delete.sim +++ b/tests/script/unique/account/account_delete.sim @@ -59,10 +59,17 @@ endi sql drop account oroot print ============= step4 -sleep 2000 +$x = 0 +show4: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + sql show dnodes -if $data02 != 0 then - return -1 +if $data03 != 0 then + goto show4 endi print ============= step5 diff --git a/tests/script/unique/account/basic.sim b/tests/script/unique/account/basic.sim index 7e386bb04f..6f64975ac2 100644 --- a/tests/script/unique/account/basic.sim +++ b/tests/script/unique/account/basic.sim @@ -1,6 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/exec_up_up.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect print =============== show accounts diff --git a/tests/script/unique/account/monitor.sim b/tests/script/unique/account/monitor.sim deleted file mode 100644 index ce71fb766e..0000000000 --- a/tests/script/unique/account/monitor.sim +++ /dev/null @@ -1,36 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/ip.sh -i 1 -s up -system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/cfg.sh -n dnode1 -c monitor -v 0 - -print ========== step1 -system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 -system sh/exec_up.sh -n dnode1 -s start - -sql connect -sleep 3000 - -print ========== step2 -sql drop database log -x step21 - return -1 -step21: -sql drop table log.cpu -x step22 - return -1 -step22: -sql drop user log -x step23 - return -1 -step23: - -print ========== step3 - -sleep 3000 -sql select * from log.dn -if $rows == 0 then - return -1 -endi - - - - diff --git a/tests/script/unique/account/paras.sim b/tests/script/unique/account/paras.sim index f312a30c14..14ee2f6c85 100644 --- a/tests/script/unique/account/paras.sim +++ b/tests/script/unique/account/paras.sim @@ -1,6 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/exec_up_up.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect print =============== show accounts diff --git a/tests/script/unique/account/testSuite.sim b/tests/script/unique/account/testSuite.sim index b48ebd9354..dedaf029f7 100644 --- a/tests/script/unique/account/testSuite.sim +++ b/tests/script/unique/account/testSuite.sim @@ -6,4 +6,7 @@ run unique/account/pass_len.sim run unique/account/user_create.sim run unique/account/user_len.sim run unique/account/authority.sim +run unique/account/basic.sim +run unique/account/paras.sim +run unique/account/usage.sim run unique/account/monitor.sim diff --git a/tests/script/unique/account/usage.sim b/tests/script/unique/account/usage.sim index 4f32e0047b..5b334374c9 100644 --- a/tests/script/unique/account/usage.sim +++ b/tests/script/unique/account/usage.sim @@ -1,6 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 -system sh/exec_up_up.sh -n dnode1 -s start +system sh/exec_up.sh -n dnode1 -s start sql connect print =============== show accounts From eef5bebc4c9683f8f0cc7012863b3ba714f82bd5 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 23 Apr 2020 16:54:03 +0800 Subject: [PATCH 11/63] fix mvn not found in CI. --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 49c94e5247..36021593d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ matrix: case $TRAVIS_OS_NAME in linux) cd ${TRAVIS_BUILD_DIR}/debug - sudo make install || travis_terminate $? + make install || travis_terminate $? pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ @@ -149,7 +149,7 @@ matrix: case $TRAVIS_OS_NAME in linux) cd ${TRAVIS_BUILD_DIR}/debug - sudo make install || travis_terminate $? + make install || travis_terminate $? pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ @@ -162,7 +162,7 @@ matrix: travis_terminate $? fi - sudo pkill taosd + pkill taosd sleep 1 cd ${TRAVIS_BUILD_DIR} From b7b8edf212cb5de7742576464618ec5a8cfdcb17 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 23 Apr 2020 17:10:27 +0800 Subject: [PATCH 12/63] [td-171] fix bugs in create table --- src/client/inc/tscUtil.h | 9 ++++-- src/client/src/tscLocal.c | 8 ++--- src/client/src/tscSQLParser.c | 40 ++++++++++++------------ src/client/src/tscSchemaUtil.c | 8 ++--- src/client/src/tscServer.c | 25 ++++++--------- src/client/src/tscSubquery.c | 4 +-- src/client/src/tscUtil.c | 56 ++++++++++++++++++++++------------ src/mnode/src/mgmtTable.c | 3 ++ src/query/src/queryExecutor.c | 5 ++- 9 files changed, 88 insertions(+), 70 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index f366277e27..e9f077fe53 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -146,9 +146,12 @@ int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFi void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex); int32_t tscGetResRowLength(SArray* pExprList); -SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + +SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t interSize); -SSqlExpr* tscSqlExprAppend(SArray* exprInfo, int16_t functionId); + +SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize); SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size); @@ -160,7 +163,7 @@ void tscSqlExprInfoDestroy(SArray* pExprInfo); SColumn* tscColumnClone(const SColumn* src); SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); -void tscColumnListAssign(SArray* dst, const SArray* src, int16_t tableIndex); +void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex); void tscColumnListDestroy(SArray* pColList); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 8272de7d16..a2006b9a13 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -247,7 +247,7 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, strncpy(f.name, "Field", TSDB_COL_NAME_LEN); SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN); rowLen += TSDB_COL_NAME_LEN; @@ -256,7 +256,7 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, strncpy(f.name, "Type", TSDB_COL_NAME_LEN); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, typeColLength); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, typeColLength); rowLen += typeColLength; @@ -265,7 +265,7 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, strncpy(f.name, "Length", TSDB_COL_NAME_LEN); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), sizeof(int32_t)); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), sizeof(int32_t)); rowLen += sizeof(int32_t); @@ -274,7 +274,7 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, strncpy(f.name, "Note", TSDB_COL_NAME_LEN); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, noteColLength); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, noteColLength); rowLen += noteColLength; return rowLen; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index fce83c8790..077339f3ee 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -649,7 +649,7 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { } SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); SColumnList ids = getColumnList(1, 0, PRIMARYKEY_TIMESTAMP_COL_INDEX); @@ -1168,7 +1168,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), sizeof(double)); addExprParams(pExpr, arithmeticExprStr, TSDB_DATA_TYPE_BINARY, strlen(arithmeticExprStr), index.tableIndex); @@ -1292,7 +1292,7 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c pQueryInfo->type = (functionId == TSDB_FUNC_TAGPRJ)? TSDB_QUERY_TYPE_STABLE_QUERY:TSDB_QUERY_TYPE_PROJECTION_QUERY; SColumnIndex index = {tableIndex, colIndex}; - return tscSqlExprInsert(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); + return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); } void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex) { @@ -1346,7 +1346,7 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, functionId, pIndex, pColSchema->type, + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionId, pIndex, pColSchema->type, pColSchema->bytes, pColSchema->bytes); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); @@ -1474,7 +1474,7 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[pColIndex->columnIndex].name); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, functionID, pColIndex, type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes); strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName)); // for all queries, the timestamp column needs to be loaded @@ -1531,7 +1531,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } else { // count the number of meters created according to the metric if (getColumnIndexByName(pToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1546,13 +1546,13 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr } int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprInsert(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); } memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); @@ -1632,7 +1632,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr if (optr == TK_DIFF) { colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); SColumnList ids = getColumnList(1, 0, 0); @@ -1644,7 +1644,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return invalidSqlErrMsg(pQueryInfo->msg, msg6); } - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, functionID, &index, resultType, resultSize, resultSize); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, resultType, resultSize, resultSize); if (optr == TK_LEASTSQUARES) { /* set the leastsquares parameters */ @@ -1836,7 +1836,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return TSDB_CODE_INVALID_SQL; } - pExpr = tscSqlExprInsert(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); @@ -1853,7 +1853,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr // set the first column ts for top/bottom query SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); const int32_t TS_COLUMN_INDEX = 0; SColumnList ids = getColumnList(1, 0, TS_COLUMN_INDEX); @@ -1862,7 +1862,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr colIndex += 1; // the first column is ts - pExpr = tscSqlExprInsert(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0); } @@ -4815,7 +4815,7 @@ void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t t if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) { SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprInsert(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); pExpr->colInfo.flag = TSDB_COL_NORMAL; // NOTE: tag column does not add to source column list @@ -4844,7 +4844,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau int16_t bytes = pSchema[index.columnIndex].bytes; char* name = pSchema[index.columnIndex].name; - pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); + pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); pExpr->colInfo.flag = TSDB_COL_TAG; // NOTE: tag column does not add to source column list @@ -4877,7 +4877,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = index}; size_t size = tscSqlExprNumOfExprs(pQueryInfo); - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_PRJ, &colIndex, pSchema->type, pSchema->bytes, pSchema->bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_PRJ, &colIndex, pSchema->type, pSchema->bytes, pSchema->bytes); pExpr->colInfo.flag = TSDB_COL_NORMAL; doLimitOutputNormalColOfGroupby(pExpr); @@ -5121,7 +5121,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { if (TSDB_COL_IS_TAG(pColIndex->flag)) { SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN); @@ -5269,7 +5269,9 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { } } - SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo->exprsInfo, TSDB_FUNC_TAG_DUMMY); + SColumnIndex ind = {0}; + SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, + tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name; strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName)); @@ -5443,7 +5445,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p if (pTagList != NULL) { // create metric[optional] for (int32_t i = 0; i < pTagList->nField; ++i) { - tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pTagList->p[col]); + tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pTagList->p[i]); } pCmd->count = pTagList->nField; diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 7ad3c08034..c898f05a77 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -161,18 +161,18 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->tableType = pTableMetaMsg->tableType; pTableMeta->tableInfo = (STableComInfo) { - .numOfTags = pTableMetaMsg->numOfTags, + .numOfTags = pTableMetaMsg->numOfTags, + .precision = pTableMetaMsg->precision, .numOfColumns = pTableMetaMsg->numOfColumns, - .precision = pTableMetaMsg->precision }; pTableMeta->sid = pTableMetaMsg->sid; pTableMeta->uid = pTableMetaMsg->uid; -// pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; + pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); - int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; for(int32_t i = 0; i < numOfTotalCols; ++i) { pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 88d0be3fb9..f2570fb229 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2169,13 +2169,11 @@ int tscProcessShowRsp(SSqlObj *pSql) { strcpy(key + 1, "showlist"); taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false); + size_t size = 0; STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - pTableMetaInfo->pTableMeta = - (STableMeta *)taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer); - - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; + pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, (char *)pTableMeta, size, tsMeterMetaKeepTimer); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if (pQueryInfo->colList == NULL) { @@ -2185,23 +2183,20 @@ int tscProcessShowRsp(SSqlObj *pSql) { SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; SColumnIndex index = {0}; - for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema = pMetaMsg->schema; + + for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { index.columnIndex = i; tscColumnListInsert(pQueryInfo->colList, &index); - TAOS_FIELD field = { - .bytes = pSchema->bytes, - .type = pSchema->type, - }; - - strncpy(field.name, pSchema->name, TSDB_COL_NAME_LEN); - tscFieldInfoAppend(pFieldInfo, &field); + TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); + SFieldSupInfo* pInfo = tscFieldInfoAppend(pFieldInfo, &f); - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); - pInfo->pSqlExpr = tscSqlExprInsert(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes); } - + + pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; tscFieldInfoUpdateOffset(pQueryInfo); tfree(pTableMeta); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index fba82657aa..d17a490d57 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -301,7 +301,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) { pQueryInfo->intervalTime = pSupporter->interval; pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; - tscColumnListAssign(pQueryInfo->colList, pSupporter->colList, 0); + tscColumnListCopy(pQueryInfo->colList, pSupporter->colList, 0); tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); pQueryInfo->exprsInfo = tscSqlExprCopy(pSupporter->exprsInfo, pSupporter->uid, false); @@ -856,7 +856,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu pCol->colIndex.tableIndex = 0; } - tscColumnListAssign(pSupporter->colList, pNewQueryInfo->colList, 0); + tscColumnListCopy(pSupporter->colList, pNewQueryInfo->colList, 0); pSupporter->exprsInfo = tscSqlExprCopy(pNewQueryInfo->exprsInfo, pSupporter->uid, false); tscFieldInfoCopy(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d7db533e7d..18a39acc0f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -946,19 +946,12 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { memset(pFieldInfo, 0, sizeof(SFieldInfo)); } -SSqlExpr* tscSqlExprAppend(SArray* exprInfo, int16_t functionId) { - SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); - pExpr->functionId = functionId; - return taosArrayPush(exprInfo, &pExpr); -} - -SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, +static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t interSize) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); - - int16_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); + pExpr->functionId = functionId; // set the correct column index @@ -968,8 +961,9 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIn SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); pExpr->colInfo.colId = pSchema->colId; } - + // tag columns require the column index revised. + int16_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); if (pColIndex->columnIndex >= numOfCols) { pExpr->colInfo.flag = TSDB_COL_TAG; } else { @@ -979,13 +973,31 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIn pExpr->colInfo.flag = TSDB_COL_TAG; } } - - pExpr->colInfo.colIndex = pColIndex->columnIndex; - pExpr->resType = type; - pExpr->resBytes = size; - pExpr->interResBytes = interSize; - pExpr->uid = pTableMetaInfo->pTableMeta->uid; + pExpr->colInfo.colIndex = pColIndex->columnIndex; + pExpr->resType = type; + pExpr->resBytes = size; + pExpr->interResBytes = interSize; + pExpr->uid = pTableMetaInfo->pTableMeta->uid; + + return pExpr; +} + +SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize) { + int32_t num = taosArrayGetSize(pQueryInfo->exprsInfo); + if (index == num) { + return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, interSize); + } + + SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize); + taosArrayInsert(pQueryInfo->exprsInfo, index, &pExpr); + return pExpr; +} + +SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, + int16_t size, int16_t interSize) { + SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize); taosArrayPush(pQueryInfo->exprsInfo, &pExpr); return pExpr; } @@ -1183,7 +1195,7 @@ static void tscColumnDestroy(SColumn* pCol) { free(pCol); } -void tscColumnListAssign(SArray* dst, const SArray* src, int16_t tableIndex) { +void tscColumnListCopy(SArray* dst, const SArray* src, int16_t tableIndex) { if (src == NULL) { return; } @@ -1764,14 +1776,18 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void memcpy(pNewQueryInfo, pQueryInfo, sizeof(SQueryInfo)); - memset(&pNewQueryInfo->colList, 0, sizeof(pNewQueryInfo->colList)); memset(&pNewQueryInfo->fieldsInfo, 0, sizeof(SFieldInfo)); pNewQueryInfo->pTableMetaInfo = NULL; pNewQueryInfo->defaultVal = NULL; pNewQueryInfo->numOfTables = 0; pNewQueryInfo->tsBuf = NULL; - + + pNewQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pNewQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); + pNewQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); + pNewQueryInfo->exprsInfo = taosArrayInit(4, POINTER_BYTES); + tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { @@ -1785,7 +1801,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void return NULL; } - tscColumnListAssign(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); + tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); // set the correct query type if (pPrevSql != NULL) { diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 79886f734d..07e510243f 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -730,6 +730,9 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { SSchema *tschema = pStable->schema; tschema[col].colId = pStable->nextColId++; tschema[col].bytes = htons(tschema[col].bytes); + + // todo 1. check the length of each column; 2. check the total length of all columns + assert(tschema[col].type >= TSDB_DATA_TYPE_BOOL && tschema[col].type <= TSDB_DATA_TYPE_NCHAR); } SSdbOper oper = { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 02580e6e87..f7a82e4040 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -1306,8 +1306,6 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY pTWAInfo->EKey = pQuery->window.ekey; } - // pCtx->ptsList = tsCol; - } else if (functionId == TSDB_FUNC_ARITHM) { pCtx->param[1].pz = param; } @@ -3781,7 +3779,8 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, STableDataInfo * SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pTableDataInfo->pTableQInfo; SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; - + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { From 82241b3725a996c8b50373371ac019664028eacd Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 23 Apr 2020 17:44:36 +0800 Subject: [PATCH 13/63] [td-171] fix bugs in time filter in super table query. --- cmake/platform.inc | 2 +- src/client/src/tscSubquery.c | 2 +- src/query/src/queryExecutor.c | 16 +++++++--------- src/tsdb/src/tsdbRead.c | 21 ++++++++------------- src/util/src/talgo.c | 6 +++--- 5 files changed, 20 insertions(+), 27 deletions(-) diff --git a/cmake/platform.inc b/cmake/platform.inc index 2e0e2d6af0..1772a83fb2 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -62,7 +62,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) - ADD_DEFINITIONS(-D_M_X64) + ADD_DEFINITIONS(-D_M_X64 -D_DEBUG_VIEW) MESSAGE(STATUS "The current platform is Linux 64-bit") ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) IF (TD_ARM) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index d17a490d57..b6d13554bb 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1355,7 +1355,7 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } #ifdef _DEBUG_VIEW - printf("received data from vnode: %d rows\n", pRes->numOfRows); + printf("received data from vnode: %"PRIu64" rows\n", pRes->numOfRows); SSrcColumnInfo colInfo[256] = {0}; tscGetSrcColumnInfo(colInfo, pQueryInfo); diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index f7a82e4040..1f4ab09419 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -1036,9 +1036,8 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { TSKEY key = *(TSKEY *)(pCtx[0].aInputElemBuf + TSDB_KEYSIZE * offset); #if defined(_DEBUG_VIEW) - printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 - ", tag:%d, id:%s, query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag, pRuntimeEnv->pTabObj->meterId, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, + printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); #endif @@ -2312,8 +2311,7 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl */ if (!needToLoadDataBlock(pQuery, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { #if defined(_DEBUG_VIEW) - qTrace("QInfo:%p fileId:%d, slot:%d, block discarded by per-filter", GET_QINFO_ADDR(pQuery), pQuery->fileId, - pQuery->slot); + qTrace("QInfo:%p block discarded by per-filter", GET_QINFO_ADDR(pRuntimeEnv)); #endif // return DISK_DATA_DISCARDED; } @@ -2946,7 +2944,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { int64_t endt = taosGetTimestampMs(); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); #endif qTrace("QInfo:%p result merge completed, elapsed time:%" PRId64 " ms", GET_QINFO_ADDR(pQuery), endt - startt); @@ -4337,7 +4335,7 @@ static int64_t queryOnDataBlocks(SQInfo *pQInfo) { SDataStatis *pStatis = NULL; SArray * pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, &blockInfo, &pStatis); - TSKEY nextKey = blockInfo.window.ekey; + TSKEY nextKey = blockInfo.window.skey; if (!isIntervalQuery(pQuery)) { setExecutionContext(pQInfo, pTableQueryInfo, pTable, pTableDataInfo->groupIdx, nextKey); } else { // interval query @@ -4784,7 +4782,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); #endif } else { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); @@ -4845,7 +4843,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); + displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); #endif } } else { // not a interval query diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 2f644175a5..531a9e3b88 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -442,11 +442,13 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { return false; } - - SDataCols* pDataCols = pCheckInfo->pDataCols; + + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + assert(pCols->numOfPoints == pBlock->numOfPoints); + if (pCheckInfo->lastKey > pBlock->keyFirst) { cur->pos = - binarySearchForKey(pDataCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); + binarySearchForKey(pCols->cols[0].pData, pBlock->numOfPoints, pCheckInfo->lastKey, pQueryHandle->order); } else { cur->pos = 0; } @@ -548,8 +550,9 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf SArray* sa) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = getTrueDataBlockInfo(pCheckInfo, pBlock); - - SDataCols* pCols = pCheckInfo->pDataCols; + +// pQueryHandle->rhelper.pDataCols[0]->cols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; int32_t endPos = cur->pos; if (ASCENDING_ORDER_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { @@ -1335,10 +1338,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosArrayPush(pTableGroup, &sa); uTrace("all %d tables belong to one group", size); - -#ifdef _DEBUG_VIEW - tSidSetDisplay(pTableGroup); -#endif } else { STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); pSupp->numOfCols = numOfOrderCols; @@ -1347,10 +1346,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn); createTableGroupImpl(pTableGroup, pTableList->pData, size, pSupp, tableGroupComparFn); - -#ifdef _DEBUG_VIEW - tSidSetDisplay(pTableGroup); -#endif tfree(pSupp); } diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 76de87e67d..32978453fc 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -42,9 +42,9 @@ static void median(void *src, size_t size, size_t s, size_t e, const void *param assert(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0); #ifdef _DEBUG_VIEW - tTagsPrints(src[s], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); - tTagsPrints(src[mid], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); - tTagsPrints(src[e], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); +// tTagsPrints(src[s], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); +// tTagsPrints(src[mid], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); +// tTagsPrints(src[e], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); #endif } From 512572e88d07935cf479dec5f1d0164e3903ce1d Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 23 Apr 2020 18:03:52 +0800 Subject: [PATCH 14/63] rename tstring.c --- src/common/src/{tstring.c => tmessage.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/common/src/{tstring.c => tmessage.c} (100%) diff --git a/src/common/src/tstring.c b/src/common/src/tmessage.c similarity index 100% rename from src/common/src/tstring.c rename to src/common/src/tmessage.c From 7dc72d3f992323e36191099f3917ec5d3a17ad94 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 23 Apr 2020 18:06:15 +0800 Subject: [PATCH 15/63] [TD-91] make monitor work --- src/dnode/src/dnodeModule.c | 4 +- src/dnode/src/dnodeShell.c | 11 +- src/mnode/src/mgmtSdb.c | 4 +- src/plugins/monitor/inc/monitorSystem.h | 39 ++- src/plugins/monitor/src/monitorSystem.c | 313 +++++++++++------------- 5 files changed, 177 insertions(+), 194 deletions(-) diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 815ecb089b..aa53dddbe4 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -90,7 +90,7 @@ void dnodeCleanUpModules() { int32_t dnodeInitModules() { dnodeAllocModules(); - for (int32_t module = 0; module < TSDB_MOD_MAX; ++module) { + for (EModuleType module = 0; module < TSDB_MOD_MAX; ++module) { if (tsModule[module].initFp) { if ((*tsModule[module].initFp)() != 0) { dError("failed to init module:%s", tsModule[module].name); @@ -103,7 +103,7 @@ int32_t dnodeInitModules() { } void dnodeStartModules() { - for (int32_t module = 1; module < TSDB_MOD_MAX; ++module) { + for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) { if (tsModule[module].enable && tsModule[module].startFp) { if ((*tsModule[module].startFp)() != 0) { dError("failed to start module:%s", tsModule[module].name); diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index b1fe0594bc..8a846b8a50 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -20,16 +20,17 @@ #include "taosmsg.h" #include "trpc.h" #include "tglobal.h" +#include "http.h" #include "dnode.h" #include "dnodeLog.h" #include "dnodeRead.h" #include "dnodeWrite.h" #include "dnodeShell.h" -static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessMsgFromShell(SRpcMsg *pMsg); -static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); -static void *tsDnodeShellRpc = NULL; +static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); +static void dnodeProcessMsgFromShell(SRpcMsg *pMsg); +static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); +static void * tsDnodeShellRpc = NULL; static int32_t tsDnodeQueryReqNum = 0; static int32_t tsDnodeSubmitReqNum = 0; @@ -110,7 +111,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char SDnodeStatisInfo dnodeGetStatisInfo() { SDnodeStatisInfo info = {0}; if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) { - //info.httpReqNum = httpGetReqCount(); + info.httpReqNum = httpGetReqCount(); info.queryReqNum = atomic_exchange_32(&tsDnodeQueryReqNum, 0); info.submitReqNum = atomic_exchange_32(&tsDnodeSubmitReqNum, 0); } diff --git a/src/mnode/src/mgmtSdb.c b/src/mnode/src/mgmtSdb.c index 0aae756529..7359f95b61 100644 --- a/src/mnode/src/mgmtSdb.c +++ b/src/mnode/src/mgmtSdb.c @@ -333,7 +333,7 @@ void sdbIncRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); atomic_add_fetch_32(pRefCount, 1); - if (1 && strcmp(pTable->tableName, "accounts") == 0) { + if (0 && strcmp(pTable->tableName, "accounts") == 0) { sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } @@ -345,7 +345,7 @@ void sdbDecRef(void *handle, void *pRow) { SSdbTable *pTable = handle; int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); - if (1 && strcmp(pTable->tableName, "accounts") == 0) { + if (0 && strcmp(pTable->tableName, "accounts") == 0) { sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow), *pRefCount); } diff --git a/src/plugins/monitor/inc/monitorSystem.h b/src/plugins/monitor/inc/monitorSystem.h index dc49071b16..91aa4d2ad6 100644 --- a/src/plugins/monitor/inc/monitorSystem.h +++ b/src/plugins/monitor/inc/monitorSystem.h @@ -23,17 +23,34 @@ extern "C" { #include #include -int32_t monitorInitSystem(); -int32_t monitorStartSystem(); -void monitorStopSystem(); -void monitorCleanUpSystem(); -void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t maxPointsPerSecond, - int64_t totalTimeSeries, int64_t maxTimeSeries, int64_t totalStorage, int64_t maxStorage, - int64_t totalQueryTime, int64_t maxQueryTime, int64_t totalInbound, int64_t maxInbound, - int64_t totalOutbound, int64_t maxOutbound, int64_t totalDbs, int64_t maxDbs, - int64_t totalUsers, int64_t maxUsers, int64_t totalStreams, int64_t maxStreams, - int64_t totalConns, int64_t maxConns, int8_t accessState); -void monitorSaveLog(int level, const char *const format, ...); +typedef struct { + char * acctId; + int64_t currentPointsPerSecond; + int64_t maxPointsPerSecond; + int64_t totalTimeSeries; + int64_t maxTimeSeries; + int64_t totalStorage; + int64_t maxStorage; + int64_t totalQueryTime; + int64_t maxQueryTime; + int64_t totalInbound; + int64_t maxInbound; + int64_t totalOutbound; + int64_t maxOutbound; + int64_t totalDbs; + int64_t maxDbs; + int64_t totalUsers; + int64_t maxUsers; + int64_t totalStreams; + int64_t maxStreams; + int64_t totalConns; + int64_t maxConns; + int8_t accessState; +} SAcctMonitorObj; + +void monitorSaveAcctLog(SAcctMonitorObj *pMonObj); +void monitorSaveLog(int32_t level, const char *const format, ...); +void monitorExecuteSQL(char *sql); #ifdef __cplusplus } diff --git a/src/plugins/monitor/src/monitorSystem.c b/src/plugins/monitor/src/monitorSystem.c index 776fdca986..a2c73fbb58 100644 --- a/src/plugins/monitor/src/monitorSystem.c +++ b/src/plugins/monitor/src/monitorSystem.c @@ -15,36 +15,33 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tlog.h" -#include "monitor.h" -#include "dnode.h" -#include "tsclient.h" #include "taosdef.h" -#include "tsystem.h" +#include "taoserror.h" +#include "tlog.h" #include "ttime.h" #include "ttimer.h" #include "tutil.h" +#include "tsystem.h" +#include "tscUtil.h" +#include "tsclient.h" +#include "dnode.h" #include "monitorSystem.h" -#define monitorError(...) \ - if (monitorDebugFlag & DEBUG_ERROR) { \ +#define monitorError(...) \ + if (monitorDebugFlag & DEBUG_ERROR) { \ taosPrintLog("ERROR MON ", 255, __VA_ARGS__); \ } -#define monitorWarn(...) \ - if (monitorDebugFlag & DEBUG_WARN) { \ +#define monitorWarn(...) \ + if (monitorDebugFlag & DEBUG_WARN) { \ taosPrintLog("WARN MON ", monitorDebugFlag, __VA_ARGS__); \ } -#define monitorTrace(...) \ - if (monitorDebugFlag & DEBUG_TRACE) { \ +#define monitorTrace(...) \ + if (monitorDebugFlag & DEBUG_TRACE) { \ taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); \ } #define monitorPrint(...) \ { taosPrintLog("MON ", 255, __VA_ARGS__); } -#define monitorLError(...) monitorError(__VA_ARGS__) -#define monitorLWarn(...) monitorWarn(__VA_ARGS__) -#define monitorLPrint(...) monitorPrint(__VA_ARGS__) - #define SQL_LENGTH 1024 #define LOG_LEN_STR 80 #define IP_LEN_STR 15 @@ -59,14 +56,14 @@ typedef enum { MONITOR_CMD_CREATE_TB_ACCT_ROOT, MONITOR_CMD_CREATE_TB_SLOWQUERY, MONITOR_CMD_MAX -} MonitorCommand; +} EMonitorCommand; typedef enum { MONITOR_STATE_UN_INIT, MONITOR_STATE_INITIALIZING, MONITOR_STATE_INITIALIZED, MONITOR_STATE_STOPPED -} MonitorState; +} EMonitorState; typedef struct { void * conn; @@ -77,89 +74,75 @@ typedef struct { char sql[SQL_LENGTH]; void * initTimer; void * diskTimer; -} MonitorConn; +} SMonitorConn; -MonitorConn *monitor = NULL; +static SMonitorConn tsMonitorConn; +static void monitorInitConn(void *para, void *unused); +static void monitorInitConnCb(void *param, TAOS_RES *result, int32_t code); +static void monitorInitDatabase(); +static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code); +static void monitorStartTimer(); +static void monitorSaveSystemInfo(); -TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), - void *param, void **taos); -void monitorInitConn(void *para, void *unused); -void monitorInitConnCb(void *param, TAOS_RES *result, int code); -void monitorInitDatabase(); -void monitorInitDatabaseCb(void *param, TAOS_RES *result, int code); -void monitorStartTimer(); -void monitorSaveSystemInfo(); -void monitorSaveLog(int level, const char *const format, ...); -void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t maxPointsPerSecond, - int64_t totalTimeSeries, int64_t maxTimeSeries, int64_t totalStorage, int64_t maxStorage, - int64_t totalQueryTime, int64_t maxQueryTime, int64_t totalInbound, int64_t maxInbound, - int64_t totalOutbound, int64_t maxOutbound, int64_t totalDbs, int64_t maxDbs, - int64_t totalUsers, int64_t maxUsers, int64_t totalStreams, int64_t maxStreams, - int64_t totalConns, int64_t maxConns, int8_t accessState); -void (*mnodeCountRequestFp)(SDnodeStatisInfo *info) = NULL; -void monitorExecuteSQL(char *sql); - -void monitorCheckDiskUsage(void *para, void *unused) { +static void monitorCheckDiskUsage(void *para, void *unused) { taosGetDisk(); - taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &monitor->diskTimer); + taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &tsMonitorConn.diskTimer); } -int monitorInitSystem() { - monitor = (MonitorConn *)malloc(sizeof(MonitorConn)); - memset(monitor, 0, sizeof(MonitorConn)); - taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &monitor->diskTimer); +int32_t monitorInitSystem() { + taos_init(); + taosTmrReset(monitorCheckDiskUsage, CHECK_INTERVAL, NULL, tscTmr, &tsMonitorConn.diskTimer); return 0; } -int monitorStartSystem() { - if (monitor == NULL) { - monitorInitSystem(); - } - taosTmrReset(monitorInitConn, 10, NULL, tscTmr, &monitor->initTimer); +int32_t monitorStartSystem() { + monitorPrint("start monitor module"); + monitorInitSystem(); + taosTmrReset(monitorInitConn, 10, NULL, tscTmr, &tsMonitorConn.initTimer); return 0; } -void monitorStartSystemRetry() { - if (monitor->initTimer != NULL) { - taosTmrReset(monitorInitConn, 3000, NULL, tscTmr, &monitor->initTimer); +static void monitorStartSystemRetry() { + if (tsMonitorConn.initTimer != NULL) { + taosTmrReset(monitorInitConn, 3000, NULL, tscTmr, &tsMonitorConn.initTimer); } } -void monitorInitConn(void *para, void *unused) { +static void monitorInitConn(void *para, void *unused) { monitorPrint("starting to initialize monitor service .."); - monitor->state = MONITOR_STATE_INITIALIZING; + tsMonitorConn.state = MONITOR_STATE_INITIALIZING; - if (monitor->privateIpStr[0] == 0) { - strcpy(monitor->privateIpStr, tsPrivateIp); - for (int i = 0; i < TSDB_IPv4ADDR_LEN; ++i) { - if (monitor->privateIpStr[i] == '.') { - monitor->privateIpStr[i] = '_'; + if (tsMonitorConn.privateIpStr[0] == 0) { + strcpy(tsMonitorConn.privateIpStr, tsPrivateIp); + for (int32_t i = 0; i < TSDB_IPv4ADDR_LEN; ++i) { + if (tsMonitorConn.privateIpStr[i] == '.') { + tsMonitorConn.privateIpStr[i] = '_'; } } } - if (monitor->conn == NULL) { - taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, monitorInitConnCb, monitor, &(monitor->conn)); + if (tsMonitorConn.conn == NULL) { + taos_connect_a(NULL, "monitor", tsInternalPass, "", 0, monitorInitConnCb, &tsMonitorConn, &(tsMonitorConn.conn)); } else { monitorInitDatabase(); } } -void monitorInitConnCb(void *param, TAOS_RES *result, int code) { +static void monitorInitConnCb(void *param, TAOS_RES *result, int32_t code) { if (code < 0) { - monitorError("monitor:%p, connect to database failed, code:%d", monitor->conn, code); - taos_close(monitor->conn); - monitor->conn = NULL; - monitor->state = MONITOR_STATE_UN_INIT; + monitorError("monitor:%p, connect to database failed, reason:%s", tsMonitorConn.conn, tstrerror(code)); + taos_close(tsMonitorConn.conn); + tsMonitorConn.conn = NULL; + tsMonitorConn.state = MONITOR_STATE_UN_INIT; monitorStartSystemRetry(); return; } - monitorTrace("monitor:%p, connect to database success, code:%d", monitor->conn, code); + monitorTrace("monitor:%p, connect to database success, reason:%s", tsMonitorConn.conn, tstrerror(code)); monitorInitDatabase(); } -void dnodeBuildMonitorSql(char *sql, int cmd) { +static void dnodeBuildMonitorSql(char *sql, int32_t cmd) { memset(sql, 0, SQL_LENGTH); if (cmd == MONITOR_CMD_CREATE_DB) { @@ -180,7 +163,7 @@ void dnodeBuildMonitorSql(char *sql, int cmd) { tsMonitorDbName, IP_LEN_STR + 1); } else if (cmd == MONITOR_CMD_CREATE_TB_DN) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn_%s using %s.dn tags('%s')", tsMonitorDbName, - monitor->privateIpStr, tsMonitorDbName, tsPrivateIp); + tsMonitorConn.privateIpStr, tsMonitorDbName, tsPrivateIp); } else if (cmd == MONITOR_CMD_CREATE_MT_ACCT) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct(ts timestamp " @@ -215,111 +198,107 @@ void dnodeBuildMonitorSql(char *sql, int cmd) { sql[SQL_LENGTH] = 0; } -void monitorInitDatabase() { - if (monitor->cmdIndex < MONITOR_CMD_MAX) { - dnodeBuildMonitorSql(monitor->sql, monitor->cmdIndex); - taos_query_a(monitor->conn, monitor->sql, monitorInitDatabaseCb, NULL); +static void monitorInitDatabase() { + if (tsMonitorConn.cmdIndex < MONITOR_CMD_MAX) { + dnodeBuildMonitorSql(tsMonitorConn.sql, tsMonitorConn.cmdIndex); + taos_query_a(tsMonitorConn.conn, tsMonitorConn.sql, monitorInitDatabaseCb, NULL); } else { - monitor->state = MONITOR_STATE_INITIALIZED; + tsMonitorConn.state = MONITOR_STATE_INITIALIZED; monitorPrint("monitor service init success"); monitorStartTimer(); } } -void monitorInitDatabaseCb(void *param, TAOS_RES *result, int code) { +static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { if (-code == TSDB_CODE_TABLE_ALREADY_EXIST || -code == TSDB_CODE_DB_ALREADY_EXIST || code >= 0) { - monitorTrace("monitor:%p, sql success, code:%d, %s", monitor->conn, code, monitor->sql); - if (monitor->cmdIndex == MONITOR_CMD_CREATE_TB_LOG) { - monitorLPrint("dnode:%s is started", tsPrivateIp); + monitorTrace("monitor:%p, sql success, reason:%d, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); + if (tsMonitorConn.cmdIndex == MONITOR_CMD_CREATE_TB_LOG) { + monitorPrint("dnode:%s is started", tsPrivateIp); } - monitor->cmdIndex++; + tsMonitorConn.cmdIndex++; monitorInitDatabase(); } else { - monitorError("monitor:%p, sql failed, code:%d, %s", monitor->conn, code, monitor->sql); - monitor->state = MONITOR_STATE_UN_INIT; + monitorError("monitor:%p, sql failed, reason:%s, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); + tsMonitorConn.state = MONITOR_STATE_UN_INIT; monitorStartSystemRetry(); } } void monitorStopSystem() { - if (monitor == NULL) { - return; - } - - monitorLPrint("dnode:%s monitor module is stopped", tsPrivateIp); - monitor->state = MONITOR_STATE_STOPPED; + monitorPrint("monitor module is stopped"); + tsMonitorConn.state = MONITOR_STATE_STOPPED; // taosLogFp = NULL; - if (monitor->initTimer != NULL) { - taosTmrStopA(&(monitor->initTimer)); + if (tsMonitorConn.initTimer != NULL) { + taosTmrStopA(&(tsMonitorConn.initTimer)); } - if (monitor->timer != NULL) { - taosTmrStopA(&(monitor->timer)); + if (tsMonitorConn.timer != NULL) { + taosTmrStopA(&(tsMonitorConn.timer)); } } void monitorCleanUpSystem() { - monitorPrint("monitor service cleanup"); monitorStopSystem(); + monitorPrint("monitor module cleanup"); } -void monitorStartTimer() { - taosTmrReset(monitorSaveSystemInfo, tsMonitorInterval * 1000, NULL, tscTmr, &monitor->timer); +static void monitorStartTimer() { + taosTmrReset(monitorSaveSystemInfo, tsMonitorInterval * 1000, NULL, tscTmr, &tsMonitorConn.timer); } -void dnodeMontiorInsertAcctCallback(void *param, TAOS_RES *result, int code) { +static void dnodeMontiorInsertAcctCallback(void *param, TAOS_RES *result, int32_t code) { if (code < 0) { - monitorError("monitor:%p, save account info failed, code:%d", monitor->conn, code); + monitorError("monitor:%p, save account info failed, code:%s", tsMonitorConn.conn, tstrerror(code)); } else if (code == 0) { - monitorError("monitor:%p, save account info failed, affect rows:%d", monitor->conn, code); + monitorError("monitor:%p, save account info failed, affect rows:%d", tsMonitorConn.conn, code); } else { - monitorTrace("monitor:%p, save account info success, code:%d", monitor->conn, code); + monitorTrace("monitor:%p, save account info success, code:%s", tsMonitorConn.conn, tstrerror(code)); } } -void dnodeMontiorInsertSysCallback(void *param, TAOS_RES *result, int code) { +static void dnodeMontiorInsertSysCallback(void *param, TAOS_RES *result, int32_t code) { if (code < 0) { - monitorError("monitor:%p, save system info failed, code:%d %s", monitor->conn, code, monitor->sql); + monitorError("monitor:%p, save system info failed, code:%s %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); } else if (code == 0) { - monitorError("monitor:%p, save system info failed, affect rows:%d %s", monitor->conn, code, monitor->sql); + monitorError("monitor:%p, save system info failed, affect rows:%d %s", tsMonitorConn.conn, code, tsMonitorConn.sql); } else { - monitorTrace("monitor:%p, save system info success, code:%d %s", monitor->conn, code, monitor->sql); + monitorTrace("monitor:%p, save system info success, code:%s %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); } } -void dnodeMontiorInsertLogCallback(void *param, TAOS_RES *result, int code) { +static void dnodeMontiorInsertLogCallback(void *param, TAOS_RES *result, int32_t code) { if (code < 0) { - monitorError("monitor:%p, save log failed, code:%d", monitor->conn, code); + monitorError("monitor:%p, save log failed, code:%s", tsMonitorConn.conn, tstrerror(code)); } else if (code == 0) { - monitorError("monitor:%p, save log failed, affect rows:%d", monitor->conn, code); + monitorError("monitor:%p, save log failed, affect rows:%d", tsMonitorConn.conn, code); } else { - monitorTrace("monitor:%p, save log info success, code:%d", monitor->conn, code); + monitorTrace("monitor:%p, save log info success, code:%s", tsMonitorConn.conn, tstrerror(code)); } } // unit is MB -int monitorBuildMemorySql(char *sql) { +static int32_t monitorBuildMemorySql(char *sql) { float sysMemoryUsedMB = 0; bool suc = taosGetSysMemory(&sysMemoryUsedMB); if (!suc) { - monitorError("monitor:%p, get sys memory info failed.", monitor->conn); + monitorError("monitor:%p, get sys memory info failed.", tsMonitorConn.conn); } float procMemoryUsedMB = 0; suc = taosGetProcMemory(&procMemoryUsedMB); if (!suc) { - monitorError("monitor:%p, get proc memory info failed.", monitor->conn); + monitorError("monitor:%p, get proc memory info failed.", tsMonitorConn.conn); } return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB); } // unit is % -int monitorBuildCpuSql(char *sql) { +static int32_t monitorBuildCpuSql(char *sql) { float sysCpuUsage = 0, procCpuUsage = 0; bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage); if (!suc) { - monitorError("monitor:%p, get cpu usage failed.", monitor->conn); + monitorError("monitor:%p, get cpu usage failed.", tsMonitorConn.conn); } if (sysCpuUsage <= procCpuUsage) { @@ -330,51 +309,45 @@ int monitorBuildCpuSql(char *sql) { } // unit is GB -int monitorBuildDiskSql(char *sql) { +static int32_t monitorBuildDiskSql(char *sql) { return sprintf(sql, ", %f, %d", (tsTotalDataDirGB - tsAvailDataDirGB), (int32_t)tsTotalDataDirGB); } // unit is Kb -int monitorBuildBandSql(char *sql) { +static int32_t monitorBuildBandSql(char *sql) { float bandSpeedKb = 0; bool suc = taosGetBandSpeed(&bandSpeedKb); if (!suc) { - monitorError("monitor:%p, get bandwidth speed failed.", monitor->conn); + monitorError("monitor:%p, get bandwidth speed failed.", tsMonitorConn.conn); } return sprintf(sql, ", %f", bandSpeedKb); } -int monitorBuildReqSql(char *sql) { - SDnodeStatisInfo info; - info.httpReqNum = info.submitReqNum = info.queryReqNum = 0; - (*mnodeCountRequestFp)(&info); - +static int32_t monitorBuildReqSql(char *sql) { + SDnodeStatisInfo info = dnodeGetStatisInfo(); return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum); } -int monitorBuildIoSql(char *sql) { +static int32_t monitorBuildIoSql(char *sql) { float readKB = 0, writeKB = 0; bool suc = taosGetProcIO(&readKB, &writeKB); if (!suc) { - monitorError("monitor:%p, get io info failed.", monitor->conn); + monitorError("monitor:%p, get io info failed.", tsMonitorConn.conn); } return sprintf(sql, ", %f, %f", readKB, writeKB); } -void monitorSaveSystemInfo() { - if (monitor->state != MONITOR_STATE_INITIALIZED) { - return; - } - - if (mnodeCountRequestFp == NULL) { +static void monitorSaveSystemInfo() { + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) { + monitorStartTimer(); return; } int64_t ts = taosGetTimestampUs(); - char * sql = monitor->sql; - int pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, monitor->privateIpStr, ts); + char * sql = tsMonitorConn.sql; + int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, tsMonitorConn.privateIpStr, ts); pos += monitorBuildCpuSql(sql + pos); pos += monitorBuildMemorySql(sql + pos); @@ -383,65 +356,57 @@ void monitorSaveSystemInfo() { pos += monitorBuildIoSql(sql + pos); pos += monitorBuildReqSql(sql + pos); - monitorTrace("monitor:%p, save system info, sql:%s", monitor->conn, sql); - taos_query_a(monitor->conn, sql, dnodeMontiorInsertSysCallback, "log"); + monitorTrace("monitor:%p, save system info, sql:%s", tsMonitorConn.conn, sql); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertSysCallback, "log"); - if (monitor->timer != NULL && monitor->state != MONITOR_STATE_STOPPED) { + if (tsMonitorConn.timer != NULL && tsMonitorConn.state != MONITOR_STATE_STOPPED) { monitorStartTimer(); } } -void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t maxPointsPerSecond, - int64_t totalTimeSeries, int64_t maxTimeSeries, int64_t totalStorage, int64_t maxStorage, - int64_t totalQueryTime, int64_t maxQueryTime, int64_t totalInbound, int64_t maxInbound, - int64_t totalOutbound, int64_t maxOutbound, int64_t totalDbs, int64_t maxDbs, - int64_t totalUsers, int64_t maxUsers, int64_t totalStreams, int64_t maxStreams, - int64_t totalConns, int64_t maxConns, int8_t accessState) { - if (monitor == NULL) return; - if (monitor->state != MONITOR_STATE_INITIALIZED) return; +void monitorSaveAcctLog(SAcctMonitorObj *pMon) { + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; char sql[1024] = {0}; sprintf(sql, "insert into %s.acct_%s using %s.acct tags('%s') values(now" - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 - ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 ", %d)", - tsMonitorDbName, acctId, tsMonitorDbName, acctId, - currentPointsPerSecond, maxPointsPerSecond, - totalTimeSeries, maxTimeSeries, - totalStorage, maxStorage, - totalQueryTime, maxQueryTime, - totalInbound, maxInbound, - totalOutbound, maxOutbound, - totalDbs, maxDbs, - totalUsers, maxUsers, - totalStreams, maxStreams, - totalConns, maxConns, - accessState); + tsMonitorDbName, pMon->acctId, tsMonitorDbName, pMon->acctId, + pMon->currentPointsPerSecond, pMon->maxPointsPerSecond, + pMon->totalTimeSeries, pMon->maxTimeSeries, + pMon->totalStorage, pMon->maxStorage, + pMon->totalQueryTime, pMon->maxQueryTime, + pMon->totalInbound, pMon->maxInbound, + pMon->totalOutbound, pMon->maxOutbound, + pMon->totalDbs, pMon->maxDbs, + pMon->totalUsers, pMon->maxUsers, + pMon->totalStreams, pMon->maxStreams, + pMon->totalConns, pMon->maxConns, + pMon->accessState); - monitorTrace("monitor:%p, save account info, sql %s", monitor->conn, sql); - taos_query_a(monitor->conn, sql, dnodeMontiorInsertAcctCallback, "account"); + monitorTrace("monitor:%p, save account info, sql %s", tsMonitorConn.conn, sql); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertAcctCallback, "account"); } -void monitorSaveLog(int level, const char *const format, ...) { +void monitorSaveLog(int32_t level, const char *const format, ...) { va_list argpointer; char sql[SQL_LENGTH] = {0}; - int max_length = SQL_LENGTH - 30; + int32_t max_length = SQL_LENGTH - 30; - if (monitor->state != MONITOR_STATE_INITIALIZED) { - return; - } + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; - int len = snprintf(sql, (size_t)max_length, "import into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName, - taosGetTimestampUs(), level); + int32_t len = snprintf(sql, (size_t)max_length, "import into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName, + taosGetTimestampUs(), level); va_start(argpointer, format); len += vsnprintf(sql + len, (size_t)(max_length - len), format, argpointer); @@ -451,11 +416,11 @@ void monitorSaveLog(int level, const char *const format, ...) { len += sprintf(sql + len, "', '%s')", tsPrivateIp); sql[len++] = 0; - monitorTrace("monitor:%p, save log, sql: %s", monitor->conn, sql); - taos_query_a(monitor->conn, sql, dnodeMontiorInsertLogCallback, "log"); + monitorTrace("monitor:%p, save log, sql: %s", tsMonitorConn.conn, sql); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertLogCallback, "log"); } void monitorExecuteSQL(char *sql) { - monitorTrace("monitor:%p, execute sql: %s", monitor->conn, sql); - taos_query_a(monitor->conn, sql, NULL, NULL); + monitorTrace("monitor:%p, execute sql: %s", tsMonitorConn.conn, sql); + taos_query_a(tsMonitorConn.conn, sql, NULL, NULL); } From 08fd09462bd8dadedf9a8a2f417014a1a1e8786a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 23 Apr 2020 18:06:49 +0800 Subject: [PATCH 16/63] reduce valgrind threshold to 16. --- .travis.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36021593d1..db6ce79703 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,6 @@ matrix: - python3-setuptools - valgrind - before_install: - - sudo apt update -y -qq - - sudo apt install -y net-tools python-pip python-setuptools python3-pip python3-setuptools valgrind - before_script: - cd ${TRAVIS_BUILD_DIR} - mkdir debug @@ -63,11 +59,14 @@ matrix: GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' + tail -10 mem-error-out.txt + defiMemError=`grep -m 1 'definitely lost' mem-error-out.txt | awk '{print $7}'` memError=`grep -m 1 'ERROR SUMMARY' mem-error-out.txt | awk '{print $4}'` if [ -n "$memError" ]; then - if [ "$memError" -gt 23 ]; then - echo -e "${RED} ## Memory errors number valgrind reports is $memError. More than our threshold! ## ${NC} " + if [ "$memError" -gt 16 ] && [ "$defiMemError" -gt 0 ]; then + echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" travis_terminate $memError fi fi @@ -131,10 +130,6 @@ matrix: - python3-setuptools - lcov - before_install: - - sudo apt update -y -qq - - sudo apt install -y net-tools python-pip python-setuptools python3-pip python3-setuptools lcov - before_script: - cd ${TRAVIS_BUILD_DIR} - mkdir debug From d78d7886edc05c6c6c1bdcbc27bc4f5a79ba4056 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 23 Apr 2020 18:27:33 +0800 Subject: [PATCH 17/63] [td-171] remove a macro definition --- cmake/platform.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/platform.inc b/cmake/platform.inc index 1772a83fb2..2e0e2d6af0 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -62,7 +62,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) - ADD_DEFINITIONS(-D_M_X64 -D_DEBUG_VIEW) + ADD_DEFINITIONS(-D_M_X64) MESSAGE(STATUS "The current platform is Linux 64-bit") ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) IF (TD_ARM) From 1c3021cd43b024f597a117da59651120453ce3b8 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 23 Apr 2020 21:55:15 +0800 Subject: [PATCH 18/63] [TD-91] refactor monitor module --- src/dnode/inc/dnodeLog.h | 4 -- src/inc/monitor.h | 32 +++++++++- src/inc/taosmsg.h | 1 - src/mnode/inc/mgmtLog.h | 13 ++-- src/plugins/http/inc/httpLog.h | 5 -- src/plugins/monitor/inc/monitorSystem.h | 59 ------------------- .../src/{monitorSystem.c => monitorMain.c} | 11 +++- src/vnode/inc/vnodeLog.h | 4 -- src/vnode/src/vnodeMain.c | 3 + 9 files changed, 48 insertions(+), 84 deletions(-) delete mode 100644 src/plugins/monitor/inc/monitorSystem.h rename src/plugins/monitor/src/{monitorSystem.c => monitorMain.c} (98%) diff --git a/src/dnode/inc/dnodeLog.h b/src/dnode/inc/dnodeLog.h index 380e93dd7b..18a5874e44 100644 --- a/src/dnode/inc/dnodeLog.h +++ b/src/dnode/inc/dnodeLog.h @@ -39,10 +39,6 @@ extern int32_t ddebugFlag; #define dPrint(...) \ { taosPrintLog("DND ", 255, __VA_ARGS__); } -#define dLError(...) taosLogError(__VA_ARGS__) dError(__VA_ARGS__) -#define dLWarn(...) taosLogWarn(__VA_ARGS__) dWarn(__VA_ARGS__) -#define dLPrint(...) taosLogPrint(__VA_ARGS__) dPrint(__VA_ARGS__) - #ifdef __cplusplus } #endif diff --git a/src/inc/monitor.h b/src/inc/monitor.h index bb63bf63a4..b1229cca6b 100644 --- a/src/inc/monitor.h +++ b/src/inc/monitor.h @@ -22,10 +22,38 @@ extern "C" { #include +typedef struct { + char * acctId; + int64_t currentPointsPerSecond; + int64_t maxPointsPerSecond; + int64_t totalTimeSeries; + int64_t maxTimeSeries; + int64_t totalStorage; + int64_t maxStorage; + int64_t totalQueryTime; + int64_t maxQueryTime; + int64_t totalInbound; + int64_t maxInbound; + int64_t totalOutbound; + int64_t maxOutbound; + int64_t totalDbs; + int64_t maxDbs; + int64_t totalUsers; + int64_t maxUsers; + int64_t totalStreams; + int64_t maxStreams; + int64_t totalConns; + int64_t maxConns; + int8_t accessState; +} SAcctMonitorObj; + int32_t monitorInitSystem(); int32_t monitorStartSystem(); -void monitorStopSystem(); -void monitorCleanUpSystem(); +void monitorStopSystem(); +void monitorCleanUpSystem(); +void monitorSaveAcctLog(SAcctMonitorObj *pMonObj); +void monitorSaveLog(int32_t level, const char *const format, ...); +void monitorExecuteSQL(char *sql); #ifdef __cplusplus } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 7d9c343a85..9f1d8033b1 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -509,7 +509,6 @@ typedef struct { int64_t pointsWritten; uint8_t status; uint8_t role; - uint8_t accessState; uint8_t replica; uint8_t reserved[5]; } SVnodeLoad; diff --git a/src/mnode/inc/mgmtLog.h b/src/mnode/inc/mgmtLog.h index fb4bb39828..a99fa22593 100644 --- a/src/mnode/inc/mgmtLog.h +++ b/src/mnode/inc/mgmtLog.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "tlog.h" +#include "monitor.h" extern int32_t mdebugFlag; extern int32_t sdbDebugFlag; @@ -41,9 +42,9 @@ extern int32_t sdbDebugFlag; #define mPrint(...) \ { taosPrintLog("MND ", 255, __VA_ARGS__); } -#define mLError(...) mError(__VA_ARGS__) -#define mLWarn(...) mWarn(__VA_ARGS__) -#define mLPrint(...) mPrint(__VA_ARGS__) +#define mLError(...) monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) +#define mLWarn(...) monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) +#define mLPrint(...) monitorSaveLog(0, __VA_ARGS__); mPrint(__VA_ARGS__) #define sdbError(...) \ if (sdbDebugFlag & DEBUG_ERROR) { \ @@ -60,9 +61,9 @@ extern int32_t sdbDebugFlag; #define sdbPrint(...) \ { taosPrintLog("MND-SDB ", 255, __VA_ARGS__); } -#define sdbLError(...) sdbError(__VA_ARGS__) -#define sdbLWarn(...) sdbWarn(__VA_ARGS__) -#define sdbLPrint(...) sdbPrint(__VA_ARGS__) +#define sdbLError(...) monitorSaveLog(2, __VA_ARGS__); sdbError(__VA_ARGS__) +#define sdbLWarn(...) monitorSaveLog(1, __VA_ARGS__); sdbWarn(__VA_ARGS__) +#define sdbLPrint(...) monitorSaveLog(0, __VA_ARGS__); sdbPrint(__VA_ARGS__) #ifdef __cplusplus } diff --git a/src/plugins/http/inc/httpLog.h b/src/plugins/http/inc/httpLog.h index 1723c30ab9..19fa46bd23 100644 --- a/src/plugins/http/inc/httpLog.h +++ b/src/plugins/http/inc/httpLog.h @@ -39,9 +39,4 @@ extern int32_t httpDebugFlag; #define httpPrint(...) \ { taosPrintLog("HTP ", 255, __VA_ARGS__); } -#define httpLError(...) taosLogError(__VA_ARGS__) httpError(__VA_ARGS__) -#define httpLWarn(...) taosLogWarn(__VA_ARGS__) httpWarn(__VA_ARGS__) -#define httpLPrint(...) taosLogPrint(__VA_ARGS__) httpPrint(__VA_ARGS__) - - #endif diff --git a/src/plugins/monitor/inc/monitorSystem.h b/src/plugins/monitor/inc/monitorSystem.h deleted file mode 100644 index 91aa4d2ad6..0000000000 --- a/src/plugins/monitor/inc/monitorSystem.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_MONITOR_SYSTEM_H -#define TDENGINE_MONITOR_SYSTEM_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -typedef struct { - char * acctId; - int64_t currentPointsPerSecond; - int64_t maxPointsPerSecond; - int64_t totalTimeSeries; - int64_t maxTimeSeries; - int64_t totalStorage; - int64_t maxStorage; - int64_t totalQueryTime; - int64_t maxQueryTime; - int64_t totalInbound; - int64_t maxInbound; - int64_t totalOutbound; - int64_t maxOutbound; - int64_t totalDbs; - int64_t maxDbs; - int64_t totalUsers; - int64_t maxUsers; - int64_t totalStreams; - int64_t maxStreams; - int64_t totalConns; - int64_t maxConns; - int8_t accessState; -} SAcctMonitorObj; - -void monitorSaveAcctLog(SAcctMonitorObj *pMonObj); -void monitorSaveLog(int32_t level, const char *const format, ...); -void monitorExecuteSQL(char *sql); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/plugins/monitor/src/monitorSystem.c b/src/plugins/monitor/src/monitorMain.c similarity index 98% rename from src/plugins/monitor/src/monitorSystem.c rename to src/plugins/monitor/src/monitorMain.c index a2c73fbb58..dade813d9c 100644 --- a/src/plugins/monitor/src/monitorSystem.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -25,7 +25,7 @@ #include "tscUtil.h" #include "tsclient.h" #include "dnode.h" -#include "monitorSystem.h" +#include "monitor.h" #define monitorError(...) \ if (monitorDebugFlag & DEBUG_ERROR) { \ @@ -228,7 +228,6 @@ static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { void monitorStopSystem() { monitorPrint("monitor module is stopped"); tsMonitorConn.state = MONITOR_STATE_STOPPED; - // taosLogFp = NULL; if (tsMonitorConn.initTimer != NULL) { taosTmrStopA(&(tsMonitorConn.initTimer)); } @@ -399,6 +398,8 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) { } void monitorSaveLog(int32_t level, const char *const format, ...) { + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; + va_list argpointer; char sql[SQL_LENGTH] = {0}; int32_t max_length = SQL_LENGTH - 30; @@ -421,6 +422,10 @@ void monitorSaveLog(int32_t level, const char *const format, ...) { } void monitorExecuteSQL(char *sql) { + if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; + monitorTrace("monitor:%p, execute sql: %s", tsMonitorConn.conn, sql); - taos_query_a(tsMonitorConn.conn, sql, NULL, NULL); + + // bug while insert binary + // taos_query_a(tsMonitorConn.conn, sql, NULL, NULL); } diff --git a/src/vnode/inc/vnodeLog.h b/src/vnode/inc/vnodeLog.h index 5650321fdb..450220e17f 100644 --- a/src/vnode/inc/vnodeLog.h +++ b/src/vnode/inc/vnodeLog.h @@ -39,10 +39,6 @@ extern int32_t ddebugFlag; #define dPrint(...) \ { taosPrintLog("DND ", 255, __VA_ARGS__); } -#define dLError(...) taosLogError(__VA_ARGS__) dError(__VA_ARGS__) -#define dLWarn(...) taosLogWarn(__VA_ARGS__) dWarn(__VA_ARGS__) -#define dLPrint(...) taosLogPrint(__VA_ARGS__) dPrint(__VA_ARGS__) - #ifdef __cplusplus } #endif diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 6936dc0345..26791ae77c 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -332,6 +332,9 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; pLoad->vgId = htonl(pVnode->vgId); + pLoad->totalStorage = htobe64(pLoad->totalStorage); + pLoad->compStorage = htobe64(pLoad->compStorage); + pLoad->pointsWritten = htobe64(pLoad->pointsWritten); pLoad->status = pVnode->status; pLoad->role = pVnode->role; pLoad->replica = pVnode->syncCfg.replica; From 96c4e414b4b8f0119fe89df6263c5535c9d244b6 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Thu, 23 Apr 2020 22:46:30 +0800 Subject: [PATCH 19/63] [td-171] suppress create if not exists dbname return error msg --- src/client/src/tscSQLParser.c | 1 + src/client/src/tscServer.c | 19 +++++++++----- src/inc/taosmsg.h | 8 +++--- src/mnode/inc/mgmtDef.h | 28 +++++++++++++++++++- src/mnode/src/mgmtDb.c | 49 ++++++++++++++++++++++++++++------- src/query/inc/qsqlparser.h | 7 +++-- src/query/src/qparserImpl.c | 3 +-- src/query/src/queryExecutor.c | 2 +- 8 files changed, 89 insertions(+), 28 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 077339f3ee..3abbe346ac 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4784,6 +4784,7 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->rowsInFileBlock = htonl(pCreateDb->rowPerFileBlock); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); pMsg->replications = pCreateDb->replica; + pMsg->ignoreExist = pCreateDb->ignoreExists; } int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f2570fb229..bab66bcc18 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -380,7 +380,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) { int doProcessSql(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - + int32_t code = TSDB_CODE_SUCCESS; + if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_RETRIEVE || @@ -389,10 +390,15 @@ int doProcessSql(SSqlObj *pSql) { pCmd->command == TSDB_SQL_HB || pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_STABLEVGROUP) { - tscBuildMsg[pCmd->command](pSql, NULL); + pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); + } + + if (pRes->code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pSql); + return pRes->code; } - int32_t code = tscSendMsgToServer(pSql); + code = tscSendMsgToServer(pSql); if (code != TSDB_CODE_SUCCESS) { pRes->code = code; tscQueueAsyncRes(pSql); @@ -701,18 +707,19 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); char *pMsg = (char *)(pQueryMsg->colList) + numOfCols * sizeof(SColumnInfo); SSchema *pSchema = tscGetTableSchema(pTableMeta); - + + int32_t total = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); for (int32_t i = 0; i < numOfCols; ++i) { SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; - if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || + if (pCol->colIndex.columnIndex >= total || pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR) { tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex, pColSchema->name); - return -1; // 0 means build msg failed + return TSDB_CODE_INVALID_SQL; } pQueryMsg->colList[i].colId = htons(pColSchema->colId); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 7d9c343a85..636d5b863b 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -515,8 +515,8 @@ typedef struct { } SVnodeLoad; typedef struct { - char acct[TSDB_USER_LEN + 1]; - char db[TSDB_DB_NAME_LEN + 1]; + char acct[TSDB_USER_LEN]; + char db[TSDB_DB_NAME_LEN]; uint32_t vgId; int32_t maxSessions; int32_t cacheBlockSize; @@ -537,8 +537,8 @@ typedef struct { int8_t repStrategy; int8_t loadLatest; // load into mem or not uint8_t precision; // time resolution - int8_t reserved[16]; -} SDbCfg, SCMCreateDbMsg, SCMAlterDbMsg; + int8_t ignoreExist; +} SCMCreateDbMsg, SCMAlterDbMsg; typedef struct { char db[TSDB_TABLE_ID_LEN + 1]; diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index 7ed4507ee2..5599ad2e4f 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -29,6 +29,32 @@ struct SAcctObj; struct SUserObj; struct SMnodeObj; +typedef struct { + char acct[TSDB_USER_LEN]; + char db[TSDB_DB_NAME_LEN]; + uint32_t vgId; + int32_t maxSessions; + int32_t cacheBlockSize; + union { + int32_t totalBlocks; + float fraction; + } cacheNumOfBlocks; + int32_t daysPerFile; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t daysToKeep; + int32_t commitTime; + int32_t rowsInFileBlock; + int16_t blocksPerTable; + int8_t compression; + int8_t commitLog; + int8_t replications; + int8_t repStrategy; + int8_t loadLatest; // load into mem or not + uint8_t precision; // time resolution + int8_t reserved[16]; +} SDbCfg; + typedef struct SDnodeObj { int32_t dnodeId; uint32_t privateIp; @@ -53,7 +79,7 @@ typedef struct SDnodeObj { int32_t refCount; uint32_t moduleStatus; uint32_t lastReboot; // time stamp for last reboot - float score; // calc in balance function + float score; // calc in balance function float diskAvailable; // from dnode status msg int16_t diskAvgUsage; // calc from sys.disk int16_t cpuAvgUsage; // calc from sys.cpu diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index d57b75501a..61b5c985c6 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" + #include "taoserror.h" #include "tutil.h" #include "tgrant.h" @@ -300,7 +301,12 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { SDbObj *pDb = mgmtGetDb(pCreate->db); if (pDb != NULL) { mgmtDecDbRef(pDb); - return TSDB_CODE_DB_ALREADY_EXIST; + + if (pCreate->ignoreExist) { + return TSDB_CODE_SUCCESS; + } else { + return TSDB_CODE_DB_ALREADY_EXIST; + } } code = mgmtCheckDbParams(pCreate); @@ -313,18 +319,41 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { return code; } - pDb = malloc(sizeof(SDbObj)); - memset(pDb, 0, sizeof(SDbObj)); - strcpy(pDb->name, pCreate->db); - strcpy(pCreate->acct, pAcct->user); + pDb = calloc(1, sizeof(SDbObj)); + + strncpy(pDb->name, pCreate->db, TSDB_DB_NAME_LEN); + strncpy(pCreate->acct, pAcct->user, TSDB_USER_LEN); + pDb->createdTime = taosGetTimestampMs(); - pDb->cfg = *pCreate; + + pDb->cfg = (SDbCfg) { + .vgId = pCreate->vgId, + .precision = pCreate->precision, + .maxSessions = pCreate->maxSessions, + .cacheNumOfBlocks.totalBlocks = pCreate->cacheNumOfBlocks.totalBlocks, + .rowsInFileBlock = pCreate->rowsInFileBlock, + .commitLog = pCreate->commitLog, + .replications = pCreate->replications, + .daysPerFile = pCreate->daysPerFile, + .cacheBlockSize = pCreate->cacheBlockSize, + .commitTime = pCreate->commitTime, + .blocksPerTable = pCreate->blocksPerTable, + .compression = pCreate->compression, + .daysToKeep = pCreate->daysToKeep, + .daysToKeep1 = pCreate->daysToKeep1, + .daysToKeep2 = pCreate->daysToKeep2, + .loadLatest = pCreate->loadLatest, + .repStrategy = pCreate->repStrategy, + }; + + strncpy(pDb->cfg.db, pCreate->db, TSDB_DB_NAME_LEN); + strncpy(pDb->cfg.acct, pCreate->acct, TSDB_USER_LEN); SSdbOper oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDbSdb, - .pObj = pDb, - .rowSize = sizeof(SDbObj) + .type = SDB_OPER_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, + .rowSize = sizeof(SDbObj), }; code = sdbInsertRow(&oper); diff --git a/src/query/inc/qsqlparser.h b/src/query/inc/qsqlparser.h index b1799b6902..42dda2308f 100644 --- a/src/query/inc/qsqlparser.h +++ b/src/query/inc/qsqlparser.h @@ -114,14 +114,13 @@ typedef struct SCreateDBInfo { int32_t tablesPerVnode; int32_t daysPerFile; int32_t rowPerFileBlock; - - float numOfAvgCacheBlocks; - int32_t numOfBlocksPerTable; - + float numOfAvgCacheBlocks; + int32_t numOfBlocksPerTable; int64_t commitTime; int32_t commitLog; int32_t compressionLevel; SSQLToken precision; + bool ignoreExists; tVariantList *keep; } SCreateDBInfo; diff --git a/src/query/src/qparserImpl.c b/src/query/src/qparserImpl.c index 28b30f6549..075dbc9d14 100644 --- a/src/query/src/qparserImpl.c +++ b/src/query/src/qparserImpl.c @@ -815,8 +815,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt = *pDB; pInfo->pDCLInfo->dbOpt.dbname = *pToken; - - tTokenListAppend(pInfo->pDCLInfo, pIgExists); + pInfo->pDCLInfo->dbOpt.ignoreExists = (pIgExists != NULL); } void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 1f4ab09419..b2c814d101 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -3555,7 +3555,7 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * int32_t setAdditionalInfo(SQInfo *pQInfo, STable *pTable, STableQueryInfo *pTableQueryInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - assert(pTableQueryInfo->lastKey > 0); + assert(pTableQueryInfo->lastKey >= 0); setTagVal(pRuntimeEnv, pTable->tableId, pQInfo->tsdb); From 4f09cb3b8f1aff78359bbb53637ae7cb406ebb05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2020 19:08:13 +0000 Subject: [PATCH 20/63] Bump jackson-databind in /tests/comparisonTest/opentsdb/opentsdbtest Bumps [jackson-databind](https://github.com/FasterXML/jackson) from 2.9.10.3 to 2.9.10.4. - [Release notes](https://github.com/FasterXML/jackson/releases) - [Commits](https://github.com/FasterXML/jackson/commits) Signed-off-by: dependabot[bot] --- tests/comparisonTest/opentsdb/opentsdbtest/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml index b104b6112b..6c37746d24 100644 --- a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml +++ b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml @@ -118,7 +118,7 @@ com.fasterxml.jackson.core jackson-databind - 2.9.10.3 + 2.9.10.4 From 11d1a829becf35c989c1dc7a0aa76aa9f5e1fa87 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Fri, 24 Apr 2020 11:16:44 +0800 Subject: [PATCH 21/63] refactor code --- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbRWHelper.c | 40 ++++++++++++++++++------------------- src/tsdb/src/tsdbRead.c | 6 +++--- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 312258a6d9..5fd82956f7 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -245,7 +245,7 @@ typedef struct { int32_t len; int32_t offset; int32_t hasLast : 1; - int32_t numOfSuperBlocks : 31; + int32_t numOfBlocks : 31; int32_t checksum; TSKEY maxKey; } SCompIdx; /* sizeof(SCompIdx) = 24 */ diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 8b5364e362..25989a2322 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -322,12 +322,12 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { if (tsdbWriteBlockToFile(pHelper, pWFile, pDataCols, rowsToWrite, &compBlock, isLast, true) < 0) goto _err; - if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfSuperBlocks) < 0) goto _err; + if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) goto _err; } else { // (Has old data) AND ((has last block) OR (key overlap)), need to merge the block SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks), - pIdx->numOfSuperBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE); + pIdx->numOfBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE); - int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfSuperBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks); + int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks); if (pCompBlock == NULL) { // No key overlap, must has last block, just merge with the last block ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last); @@ -362,18 +362,18 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { if ((pHelper->files.nLastF.fd > 0) && (pHelper->hasOldLastBlock)) { if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; - SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + pIdx->numOfSuperBlocks - 1; + SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + pIdx->numOfBlocks - 1; ASSERT(pCompBlock->last); if (pCompBlock->numOfSubBlocks > 1) { - if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfSuperBlocks - 1), NULL) < 0) return -1; + if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfPoints > 0 && pHelper->pDataCols[0]->numOfPoints < pHelper->config.minRowsPerFileBlock); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], pHelper->pDataCols[0]->numOfPoints, &compBlock, true, true) < 0) return -1; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfSuperBlocks - 1) < 0) return -1; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; } else { if (lseek(pHelper->files.lastF.fd, pCompBlock->offset, SEEK_SET) < 0) return -1; @@ -827,7 +827,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa ASSERT(keyFirst <= blockAtIdx(pHelper, blkIdx)->keyLast); TSKEY keyLimit = - (blkIdx == pIdx->numOfSuperBlocks - 1) ? INT64_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1; + (blkIdx == pIdx->numOfBlocks - 1) ? INT64_MAX : pHelper->pCompInfo->blocks[blkIdx + 1].keyFirst - 1; // rows1: number of rows must merge in this block int rows1 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast); @@ -969,7 +969,7 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int if (tsdbAdjustInfoSizeIfNeeded(pHelper, pIdx->len + sizeof(SCompInfo)) < 0) goto _err; // Change the offset - for (int i = 0; i < pIdx->numOfSuperBlocks; i++) { + for (int i = 0; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); } @@ -984,13 +984,13 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int } pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock; - pIdx->numOfSuperBlocks++; + pIdx->numOfBlocks++; pIdx->len += sizeof(SCompBlock); ASSERT(pIdx->len <= tsizeof(pHelper->pCompInfo)); - pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; - pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; + pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; + pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; - if (pIdx->numOfSuperBlocks > 1) { + if (pIdx->numOfBlocks > 1) { ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst); } @@ -1022,7 +1022,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)), (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); - for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); } @@ -1040,7 +1040,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId pIdx->len += sizeof(SCompBlock); } else { // Need to create two sub-blocks void *ptr = NULL; - for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; if (pTCompBlock->numOfSubBlocks > 1) { ptr = (void *)((char *)(pHelper->pCompInfo) + pTCompBlock->offset + pTCompBlock->len); @@ -1053,7 +1053,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo)); if (tsize > 0) { memmove((void *)((char *)ptr + sizeof(SCompBlock) * 2), ptr, tsize); - for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += (sizeof(SCompBlock) * 2); } @@ -1074,8 +1074,8 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId pIdx->len += (sizeof(SCompBlock) * 2); } - pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; - pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; + pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; + pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; return 0; @@ -1102,7 +1102,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); } - for (int i = blkIdx + 1; i < pIdx->numOfSuperBlocks; i++) { + for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks); } @@ -1112,8 +1112,8 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int *pSCompBlock = *pCompBlock; - pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].keyLast; - pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last; + pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; + pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; return 0; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 531a9e3b88..d5003ed57c 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -301,7 +301,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid]; - if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file + if (compIndex->len == 0 || compIndex->numOfBlocks == 0) { // no data block in this file, try next file continue;//no data blocks in the file belongs to pCheckInfo->pTable } else { if (pCheckInfo->compSize < compIndex->len) { @@ -327,7 +327,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo TSKEY e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); // discard the unqualified data block based on the query time window - int32_t start = binarySearchForBlockImpl(pCompInfo->blocks, compIndex->numOfSuperBlocks, s, TSDB_ORDER_ASC); + int32_t start = binarySearchForBlockImpl(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); int32_t end = start; if (s > pCompInfo->blocks[start].keyLast) { @@ -335,7 +335,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo } // todo speedup the procedure of located end block - while (end < compIndex->numOfSuperBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { + while (end < compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { end += 1; } From e6f27e5e1798a2d48463d1d4e420ce5282b0c88e Mon Sep 17 00:00:00 2001 From: hzcheng Date: Fri, 24 Apr 2020 12:00:04 +0800 Subject: [PATCH 22/63] fix python NULL problem --- src/connector/python/linux/python2/taos/cinterface.py | 3 +++ src/connector/python/linux/python3/taos/cinterface.py | 3 +++ src/connector/python/windows/python2/taos/cinterface.py | 3 +++ src/connector/python/windows/python3/taos/cinterface.py | 3 +++ 4 files changed, 12 insertions(+) diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 505619436c..d9460efb21 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index 7fcedc9fe9..77001609b6 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index f8cdfcc51e..a18d372db2 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index b4b44e199c..cbcf2d884e 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") From 3e0bd397e9912e2c57dde03f1033ebcdbfd5b18d Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 24 Apr 2020 13:18:17 +0800 Subject: [PATCH 23/63] [TD-52] [TD-148] add cluster test script --- src/client/src/tscServer.c | 2 ++ src/client/src/tscSystem.c | 2 +- src/dnode/src/dnodeMClient.c | 17 ++++++++++++++++- src/dnode/src/dnodeMnode.c | 2 +- src/inc/dnode.h | 3 ++- src/mnode/src/mgmtDClient.c | 4 +++- src/mnode/src/mgmtDServer.c | 11 ++++++++--- src/mnode/src/mgmtShell.c | 15 ++++++++++----- tests/script/sh/deploy.sh | 2 +- tests/script/unique/mnode/mgmt33.sim | 6 +++--- tests/script/unique/mnode/mgmt34.sim | 6 +++--- 11 files changed, 50 insertions(+), 20 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f2570fb229..21980289d6 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -163,11 +163,13 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { tscGetQueryInfoDetailSafely(&pSql->cmd, 0, &pQueryInfo); pQueryInfo->command = TSDB_SQL_HB; + pSql->cmd.command = TSDB_SQL_HB; if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { tfree(pSql); return; } + pSql->cmd.command = TSDB_SQL_HB; pSql->param = pObj; pSql->pTscObj = pObj; pSql->signature = pSql; diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 6713f84f99..484c9e344e 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -67,7 +67,6 @@ int32_t tscInitRpc(const char *user, const char *secret) { rpcInit.label = "TSC-vnode"; rpcInit.numOfThreads = tscNumOfThreads; rpcInit.cfp = tscProcessMsgFromServer; - rpcInit.ufp = tscUpdateIpSet; rpcInit.sessions = tsMaxVnodeConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char*)user; @@ -96,6 +95,7 @@ int32_t tscInitRpc(const char *user, const char *secret) { rpcInit.label = "TSC-mgmt"; rpcInit.numOfThreads = 1; rpcInit.cfp = tscProcessMsgFromServer; + rpcInit.ufp = tscUpdateIpSet; rpcInit.sessions = tsMaxMgmtConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = 2000; diff --git a/src/dnode/src/dnodeMClient.c b/src/dnode/src/dnodeMClient.c index ee805a2a0c..151d44922d 100644 --- a/src/dnode/src/dnodeMClient.c +++ b/src/dnode/src/dnodeMClient.c @@ -59,6 +59,19 @@ void dnodeUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { tsMnodeIpSet = *pIpSet; } +void dnodeGetMnodeIpSet(void *ipSetRaw, bool usePublicIp) { + SRpcIpSet *ipSet = ipSetRaw; + ipSet->numOfIps = tsMnodeInfos.nodeNum; + ipSet->inUse = tsMnodeInfos.inUse; + for (int32_t i = 0; i < tsMnodeInfos.nodeNum; ++i) { + if (usePublicIp) { + ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; + } else { + ipSet->ip[i] = tsMnodeInfos.nodeInfos[i].nodeIp; + } + } +} + int32_t dnodeInitMClient() { dnodeReadDnodeCfg(); tsRebootTime = taosGetTimestampSec(); @@ -138,7 +151,9 @@ static void dnodeProcessRspFromMnode(SRpcMsg *pMsg) { if (tsDnodeProcessMgmtRspFp[pMsg->msgType]) { (*tsDnodeProcessMgmtRspFp[pMsg->msgType])(pMsg); } else { - dError("%s is not processed in mnode rpc client", taosMsg[pMsg->msgType]); + dError("%s is not processed in dnode mclient", taosMsg[pMsg->msgType]); + SRpcMsg rpcRsp = {.pCont = 0, .contLen = 0, .code = TSDB_CODE_OPS_NOT_SUPPORT, .handle = pMsg->handle}; + rpcSendResponse(&rpcRsp); } rpcFreeCont(pMsg->pCont); diff --git a/src/dnode/src/dnodeMnode.c b/src/dnode/src/dnodeMnode.c index 9672a34a9f..ac477c4de1 100644 --- a/src/dnode/src/dnodeMnode.c +++ b/src/dnode/src/dnodeMnode.c @@ -88,7 +88,7 @@ static void dnodeProcessMsgFromMnode(SRpcMsg *pMsg) { if (dnodeProcessMgmtMsgFp[pMsg->msgType]) { (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg); } else { - dError("%s is not processed in mserver", taosMsg[pMsg->msgType]); + dError("%s is not processed in dnode mserver", taosMsg[pMsg->msgType]); rspMsg.code = TSDB_CODE_MSG_NOT_PROCESSED; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 618cf79b14..6fbfa7ffa9 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -41,8 +41,9 @@ void *dnodeAllocateRqueue(void *pVnode); void dnodeFreeRqueue(void *rqueue); void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code); -bool dnodeIsFirstDeploy(); +bool dnodeIsFirstDeploy(); uint32_t dnodeGetMnodeMasteIp(); +void dnodeGetMnodeIpSet(void *ipSet, bool usePublicIp); void * dnodeGetMnodeInfos(); int32_t dnodeGetDnodeId(); diff --git a/src/mnode/src/mgmtDClient.c b/src/mnode/src/mgmtDClient.c index f547128f00..759d400eb8 100644 --- a/src/mnode/src/mgmtDClient.c +++ b/src/mnode/src/mgmtDClient.c @@ -79,7 +79,9 @@ static void mgmtProcessRspFromDnode(SRpcMsg *rpcMsg) { if (mgmtProcessDnodeRspFp[rpcMsg->msgType]) { (*mgmtProcessDnodeRspFp[rpcMsg->msgType])(rpcMsg); } else { - mError("%s is not processed in dclient", taosMsg[rpcMsg->msgType]); + mError("%s is not processed in mgmt dclient", taosMsg[rpcMsg->msgType]); + SRpcMsg rpcRsp = {.pCont = 0, .contLen = 0, .code = TSDB_CODE_OPS_NOT_SUPPORT, .handle = rpcMsg->handle}; + rpcSendResponse(&rpcRsp); } rpcFreeCont(rpcMsg->pCont); diff --git a/src/mnode/src/mgmtDServer.c b/src/mnode/src/mgmtDServer.c index b551c0eae5..9f65cd5d4a 100644 --- a/src/mnode/src/mgmtDServer.c +++ b/src/mnode/src/mgmtDServer.c @@ -108,8 +108,12 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) { bool usePublicIp = false; SRpcIpSet ipSet = {0}; - mgmtGetMnodeIpSet(&ipSet, usePublicIp); - mTrace("conn from dnode ip:%s redirect msg", taosIpStr(connInfo.clientIp)); + ipSet.port = tsMnodeDnodePort; + dnodeGetMnodeIpSet(&ipSet, usePublicIp); + mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse); + for (int32_t i = 0; i < ipSet.numOfIps; ++i) { + mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i])); + } rpcSendRedirectRsp(rpcMsg->handle, &ipSet); return; } @@ -119,7 +123,8 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) { memcpy(pMsg, rpcMsg, sizeof(SRpcMsg)); mgmtAddToDServerQueue(pMsg); } else { - mError("%s is not processed in dserver", taosMsg[rpcMsg->msgType]); + mError("%s is not processed in mgmt dserver", taosMsg[rpcMsg->msgType]); + mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_MSG_NOT_PROCESSED); rpcFreeCont(rpcMsg->pCont); } } diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 752f33db13..927bbe4ed1 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -151,8 +151,13 @@ static void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg) { bool usePublicIp = (connInfo.serverIp == tsPublicIpInt); SRpcIpSet ipSet = {0}; - mgmtGetMnodeIpSet(&ipSet, usePublicIp); - mTrace("conn from ip:%s user:%s redirect msg", taosIpStr(connInfo.clientIp), connInfo.user); + ipSet.port = tsMnodeShellPort; + dnodeGetMnodeIpSet(&ipSet, usePublicIp); + mTrace("conn from shell ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse); + for (int32_t i = 0; i < ipSet.numOfIps; ++i) { + mTrace("index:%d ip:%s", i, taosIpStr(ipSet.ip[i])); + } + rpcSendRedirectRsp(rpcMsg->handle, &ipSet); return; } @@ -429,9 +434,9 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) { connect_over: rpcRsp.code = code; if (code != TSDB_CODE_SUCCESS) { - mLError("user:%s login from %s, code:%d", connInfo.user, taosIpStr(connInfo.clientIp), code); + mLError("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); } else { - mLPrint("user:%s login from %s, code:%d", connInfo.user, taosIpStr(connInfo.clientIp), code); + mLPrint("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); rpcRsp.pCont = pConnectRsp; rpcRsp.contLen = sizeof(SCMConnectRsp); } @@ -488,7 +493,7 @@ static bool mgmtCheckMsgReadOnly(SQueuedMsg *pMsg) { } static void mgmtProcessUnSupportMsg(SRpcMsg *rpcMsg) { - mError("%s is not processed in shell", taosMsg[rpcMsg->msgType]); + mError("%s is not processed in mnode shell", taosMsg[rpcMsg->msgType]); SRpcMsg rpcRsp = { .msgType = 0, .pCont = 0, diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index b1aa7c6382..41ba3c425a 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -93,7 +93,7 @@ echo "privateIp $NODE_IP" >> $TAOS_CFG echo "dDebugFlag 199" >> $TAOS_CFG echo "mDebugFlag 199" >> $TAOS_CFG echo "sdbDebugFlag 199" >> $TAOS_CFG -echo "rpcDebugFlag 131" >> $TAOS_CFG +echo "rpcDebugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "httpDebugFlag 131" >> $TAOS_CFG diff --git a/tests/script/unique/mnode/mgmt33.sim b/tests/script/unique/mnode/mgmt33.sim index 629adad029..b2235ca6dc 100644 --- a/tests/script/unique/mnode/mgmt33.sim +++ b/tests/script/unique/mnode/mgmt33.sim @@ -116,7 +116,7 @@ sleep 8000 sql show mnodes $dnode1Role = $data3_1 -$dnode2Role = $data3_2 +$dnode2Role = $data3_4 $dnode3Role = $data3_3 print 192.168.0.1 ==> $dnode1Role print 192.168.0.2 ==> $dnode2Role @@ -138,13 +138,13 @@ sleep 10000 sql show mnodes $dnode1Role = $data3_1 -$dnode2Role = $data3_2 +$dnode2Role = $data3_4 $dnode3Role = $data3_3 print 192.168.0.1 ==> $dnode1Role print 192.168.0.2 ==> $dnode2Role print 192.168.0.3 ==> $dnode3Role -if $dnode1Role != undecided then +if $dnode1Role != offline then return -1 endi #if $dnode2Role != master then diff --git a/tests/script/unique/mnode/mgmt34.sim b/tests/script/unique/mnode/mgmt34.sim index c811728deb..1c12b2ee39 100644 --- a/tests/script/unique/mnode/mgmt34.sim +++ b/tests/script/unique/mnode/mgmt34.sim @@ -183,7 +183,7 @@ endi print ============== step7 system sh/exec_up.sh -n dnode1 -s stop -sleep 10000 +sleep 4000 sql show mnodes $dnode1Role = $data3_1 @@ -195,7 +195,7 @@ print 192.168.0.2 ==> $dnode2Role print 192.168.0.3 ==> $dnode3Role print 192.168.0.4 ==> $dnode4Role -if $dnode1Role != undecided then +if $dnode1Role != offline then return -1 endi @@ -205,7 +205,7 @@ sleep 8000 sql show mnodes $dnode1Role = $data3_1 -$dnode2Role = $data3_2 +$dnode2Role = $data3_5 $dnode3Role = $data3_3 $dnode4Role = $data3_4 print 192.168.0.1 ==> $dnode1Role From af305abdc8049d7f262460a7caee17b1e2e161ea Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 24 Apr 2020 14:10:21 +0800 Subject: [PATCH 24/63] remove db prefix from table name --- src/mnode/src/mgmtTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index fea9d069a3..cad559b110 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -1242,7 +1242,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb return NULL; } - memcpy(pCreate->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN); + mgmtExtractTableName(pTable->info.tableId, pCreate->tableId); pCreate->contLen = htonl(contLen); pCreate->vgId = htonl(pTable->vgId); pCreate->tableType = pTable->info.type; @@ -1252,7 +1252,7 @@ static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableOb pCreate->uid = htobe64(pTable->uid); if (pTable->info.type == TSDB_CHILD_TABLE) { - memcpy(pCreate->superTableId, pTable->superTable->info.tableId, TSDB_TABLE_ID_LEN + 1); + mgmtExtractTableName(pTable->superTable->info.tableId, pCreate->superTableId); pCreate->numOfColumns = htons(pTable->superTable->numOfColumns); pCreate->numOfTags = htons(pTable->superTable->numOfTags); pCreate->sversion = htonl(pTable->superTable->sversion); From 2ea4875df27922d1e7f3c524ee69066e8f7aa152 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 24 Apr 2020 15:29:46 +0800 Subject: [PATCH 25/63] [td-171] fix bugs in select * for super table. --- src/client/inc/tscUtil.h | 9 +- src/client/inc/tsclient.h | 5 +- src/client/src/tscLocal.c | 12 +- src/client/src/tscSQLParser.c | 329 +++++++++++++++++++--------------- src/client/src/tscServer.c | 53 ++++-- src/client/src/tscUtil.c | 60 +++---- src/inc/taosmsg.h | 1 + src/query/inc/queryExecutor.h | 11 +- src/query/src/queryExecutor.c | 152 +++++++++++----- 9 files changed, 381 insertions(+), 251 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index e9f077fe53..c16945bbcf 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -113,7 +113,8 @@ bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd); void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t isTag); -void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex); +//void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex); +void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index); int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); @@ -148,10 +149,10 @@ void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int32_t tscGetResRowLength(SArray* pExprList); SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize); + int16_t size, int16_t interSize, bool isTagCol); SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize); + int16_t size, int16_t interSize, bool isTagCol); SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size); @@ -198,7 +199,7 @@ int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQuer void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, int16_t numOfTags, int16_t* tags); + SVgroupsInfo* vgroupList, SArray* pTagCols); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 337b6b9628..9cc6bcb364 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -78,8 +78,7 @@ typedef struct STableMetaInfo { */ int32_t vgroupIndex; char name[TSDB_TABLE_ID_LEN]; // (super) table name - int16_t numOfTags; // total required tags in query, including groupby tags - int16_t tagColumnIndex[TSDB_MAX_TAGS]; // clause + tag projection + SArray* tagColList; // involved tag columns } STableMetaInfo; /* the structure for sql function in select clause */ @@ -221,7 +220,7 @@ typedef struct SQueryInfo { int64_t clauseLimit; // limit for current sub clause // offset value in the original sql expression, NOT sent to virtual node, only applied at client side - int64_t prjOffset; + int64_t prjOffset; } SQueryInfo; typedef struct { diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index a2006b9a13..7ace940a1e 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -247,7 +247,8 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, strncpy(f.name, "Field", TSDB_COL_NAME_LEN); SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, + TSDB_COL_NAME_LEN, false); rowLen += TSDB_COL_NAME_LEN; @@ -256,7 +257,8 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, strncpy(f.name, "Type", TSDB_COL_NAME_LEN); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, typeColLength); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, + typeColLength, false); rowLen += typeColLength; @@ -265,7 +267,8 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, strncpy(f.name, "Length", TSDB_COL_NAME_LEN); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), sizeof(int32_t)); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), + sizeof(int32_t), false); rowLen += sizeof(int32_t); @@ -274,7 +277,8 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, strncpy(f.name, "Note", TSDB_COL_NAME_LEN); pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); - pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, noteColLength); + pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, + noteColLength, false); rowLen += noteColLength; return rowLen; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3abbe346ac..df47afd0dc 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -649,7 +649,8 @@ int32_t parseIntervalClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { } SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, + TSDB_KEYSIZE, false); SColumnList ids = getColumnList(1, 0, PRIMARYKEY_TIMESTAMP_COL_INDEX); @@ -1169,7 +1170,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, - sizeof(double), sizeof(double)); + sizeof(double), sizeof(double), false); addExprParams(pExpr, arithmeticExprStr, TSDB_DATA_TYPE_BINARY, strlen(arithmeticExprStr), index.tableIndex); /* todo alias name should use the original sql string */ @@ -1270,7 +1271,16 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr) { + for (int32_t i = 0; i < pIdList->num; ++i) { + int32_t tableId = pIdList->ids[i].tableIndex; + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[tableId]; + + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + if (pIdList->ids[i].columnIndex >= numOfCols) { + continue; + } + tscColumnListInsert(pQueryInfo->colList, &(pIdList->ids[i])); } @@ -1289,37 +1299,46 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, colIndex); int16_t functionId = (int16_t)((colIndex >= numOfCols) ? TSDB_FUNC_TAGPRJ : TSDB_FUNC_PRJ); - pQueryInfo->type = (functionId == TSDB_FUNC_TAGPRJ)? TSDB_QUERY_TYPE_STABLE_QUERY:TSDB_QUERY_TYPE_PROJECTION_QUERY; - - SColumnIndex index = {tableIndex, colIndex}; - return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, pSchema->bytes); + SColumnIndex index = {.tableIndex = tableIndex,}; + + if (functionId == TSDB_FUNC_TAGPRJ) { + index.columnIndex = colIndex - tscGetNumOfColumns(pTableMeta); + + addRequiredTagColumn(pTableMetaInfo, &index); + pQueryInfo->type = TSDB_QUERY_TYPE_STABLE_QUERY; + } else { + index.columnIndex = colIndex; + pQueryInfo->type = TSDB_QUERY_TYPE_PROJECTION_QUERY; + } + + return tscSqlExprAppend(pQueryInfo, functionId, &index, pSchema->type, pSchema->bytes, + pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ); } -void addRequiredTagColumn(SQueryInfo* pQueryInfo, int32_t tagColIndex, int32_t tableIndex) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - - if (pTableMetaInfo->numOfTags == 0 || pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags - 1] < tagColIndex) { - pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags++] = tagColIndex; - } else { // find the appropriate position - for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { - if (tagColIndex > pTableMetaInfo->tagColumnIndex[i]) { - continue; - } else if (tagColIndex == pTableMetaInfo->tagColumnIndex[i]) { - break; - } else { - memmove(&pTableMetaInfo->tagColumnIndex[i + 1], &pTableMetaInfo->tagColumnIndex[i], - sizeof(pTableMetaInfo->tagColumnIndex[0]) * (pTableMetaInfo->numOfTags - i)); - - pTableMetaInfo->tagColumnIndex[i] = tagColIndex; - - pTableMetaInfo->numOfTags++; - break; - } - } - } +void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index) { + tscColumnListInsert(pTableMetaInfo->tagColList, index); +// if (pTableMetaInfo->numOfTags == 0 || pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags - 1] < tagColIndex) { +// pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags++] = tagColIndex; +// } else { // find the appropriate position +// for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { +// if (tagColIndex > pTableMetaInfo->tagColumnIndex[i]) { +// continue; +// } else if (tagColIndex == pTableMetaInfo->tagColumnIndex[i]) { +// break; +// } else { +// memmove(&pTableMetaInfo->tagColumnIndex[i + 1], &pTableMetaInfo->tagColumnIndex[i], +// sizeof(pTableMetaInfo->tagColumnIndex[0]) * (pTableMetaInfo->numOfTags - i)); +// +// pTableMetaInfo->tagColumnIndex[i] = tagColIndex; +// +// pTableMetaInfo->numOfTags++; +// break; +// } +// } +// } // plus one means tbname - assert(tagColIndex >= -1 && tagColIndex < TSDB_MAX_TAGS && pTableMetaInfo->numOfTags <= TSDB_MAX_TAGS + 1); +// assert(tagColIndex >= -1 && tagColIndex < TSDB_MAX_TAGS && pTableMetaInfo->numOfTags <= TSDB_MAX_TAGS + 1); } static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { @@ -1347,7 +1366,7 @@ static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumn void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionId, pIndex, pColSchema->type, - pColSchema->bytes, pColSchema->bytes); + pColSchema->bytes, pColSchema->bytes, flag); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); if (TSDB_COL_IS_TAG(flag)) { @@ -1357,8 +1376,10 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex insertResultField(pQueryInfo, outputColIndex, &ids, pColSchema->bytes, pColSchema->type, pColSchema->name, pExpr); pExpr->colInfo.flag = flag; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); + if (TSDB_COL_IS_TAG(flag)) { - addRequiredTagColumn(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex); + addRequiredTagColumn(pTableMetaInfo, pIndex); } } @@ -1384,7 +1405,6 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum pIndex->columnIndex = j; SColumnList ids = {0}; ids.ids[0] = *pIndex; - ids.num = 1; insertResultField(pQueryInfo, startPos + j, &ids, pSchema[j].bytes, pSchema[j].type, pSchema[j].name, pExpr); @@ -1474,7 +1494,7 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema, getRevisedName(columnName, functionID, TSDB_COL_NAME_LEN, pSchema[pColIndex->columnIndex].name); } - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); strncpy(pExpr->aliasName, columnName, tListLen(pExpr->aliasName)); // for all queries, the timestamp column needs to be loaded @@ -1531,7 +1551,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false); } else { // count the number of meters created according to the metric if (getColumnIndexByName(pToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1546,13 +1566,13 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr } int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size); + pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, size, false); } memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); @@ -1633,7 +1653,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, - TSDB_KEYSIZE); + TSDB_KEYSIZE, false); SColumnList ids = getColumnList(1, 0, 0); insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].aName, pExpr); @@ -1644,7 +1664,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return invalidSqlErrMsg(pQueryInfo->msg, msg6); } - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, resultType, resultSize, resultSize); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, resultType, resultSize, resultSize, false); if (optr == TK_LEASTSQUARES) { /* set the leastsquares parameters */ @@ -1836,7 +1856,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr return TSDB_CODE_INVALID_SQL; } - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize, false); addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); @@ -1853,7 +1873,8 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr // set the first column ts for top/bottom query SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, + TSDB_KEYSIZE, false); const int32_t TS_COLUMN_INDEX = 0; SColumnList ids = getColumnList(1, 0, TS_COLUMN_INDEX); @@ -1862,7 +1883,7 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr colIndex += 1; // the first column is ts - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, resultSize, false); addExprParams(pExpr, val, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0); } @@ -2338,10 +2359,15 @@ bool hasUnsupportFunctionsForSTableQuery(SQueryInfo* pQueryInfo) { return true; } - if (pQueryInfo->groupbyExpr.numOfGroupCols != 1 || - pQueryInfo->groupbyExpr.columnInfo[0].colIndex != TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->groupbyExpr.numOfGroupCols != 1) { invalidSqlErrMsg(pQueryInfo->msg, msg2); return true; + } else { + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); + if (pColIndex->colIndex != TSDB_TBNAME_COLUMN_INDEX) { + invalidSqlErrMsg(pQueryInfo->msg, msg2); + return true; + } } } @@ -2385,72 +2411,55 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { } void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); +// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); +// +// // update tags column index for expression +// size_t size = tscSqlExprNumOfExprs(pQueryInfo); +// for (int32_t i = 0; i < size; ++i) { +// SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); +// +// if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue +// continue; +// } +// +// // not belongs to this table +// if (pExpr->uid != pTableMetaInfo->pTableMeta->uid) { +// continue; +// } - /* - * update tags column index for group by tags - * group by columns belong to this table - */ - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 && pQueryInfo->groupbyExpr.tableIndex == tableIndex) { - for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - int32_t index = pQueryInfo->groupbyExpr.columnInfo[i].colIndex; - - for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { - int32_t tagColIndex = pTableMetaInfo->tagColumnIndex[j]; - if (tagColIndex == index) { - pQueryInfo->groupbyExpr.columnInfo[i].colIndex = j; - break; - } - } - } - } - - // update tags column index for expression - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - - if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue - continue; - } - - // not belongs to this table - if (pExpr->uid != pTableMetaInfo->pTableMeta->uid) { - continue; - } - - for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { - if (pExpr->colInfo.colIndex == pTableMetaInfo->tagColumnIndex[j]) { - pExpr->colInfo.colIndex = j; - break; - } - } - } +// for (int32_t j = 0; j < pTableMetaInfo->numOfTags; ++j) { +// if (pExpr->colInfo.colIndex == pTableMetaInfo->tagColumnIndex[j]) { +// pExpr->colInfo.colIndex = j; +// break; +// } +// } +// } // update join condition tag column index - SJoinInfo* pJoinInfo = &pQueryInfo->tagCond.joinInfo; - if (!pJoinInfo->hasJoin) { // not join query - return; - } - - assert(pJoinInfo->left.uid != pJoinInfo->right.uid); - - // the join condition expression node belongs to this table(super table) - if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->left.uid) { - for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { - if (pJoinInfo->left.tagCol == pTableMetaInfo->tagColumnIndex[i]) { - pJoinInfo->left.tagCol = i; - } - } - } - - if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->right.uid) { - for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { - if (pJoinInfo->right.tagCol == pTableMetaInfo->tagColumnIndex[i]) { - pJoinInfo->right.tagCol = i; - } - } - } +// SJoinInfo* pJoinInfo = &pQueryInfo->tagCond.joinInfo; +// if (!pJoinInfo->hasJoin) { // not join query +// return; +// } +// +// assert(pJoinInfo->left.uid != pJoinInfo->right.uid); +// +// // the join condition expression node belongs to this table(super table) +// assert(0); +// if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->left.uid) { +// for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { +// if (pJoinInfo->left.tagCol == pTableMetaInfo->tagColumnIndex[i]) { +// pJoinInfo->left.tagCol = i; +// } +// } +// } +// +// if (pTableMetaInfo->pTableMeta->uid == pJoinInfo->right.uid) { +// for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { +// if (pJoinInfo->right.tagCol == pTableMetaInfo->tagColumnIndex[i]) { +// pJoinInfo->right.tagCol = i; +// } +// } +// } } int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) { @@ -2478,7 +2487,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* SSchema s = tscGetTbnameColumnSchema(); int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; - + for (int32_t i = 0; i < pList->nExpr; ++i) { tVariant* pVar = &pList->a[i].pVar; SSQLToken token = {pVar->nLen, pVar->nType, pVar->pz}; @@ -2496,7 +2505,8 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); pTableMeta = pTableMetaInfo->pTableMeta; - + + int32_t numOfCols = tscGetNumOfColumns(pTableMeta); if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pSchema = &s; } else { @@ -2504,23 +2514,31 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* } bool groupTag = false; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= tscGetNumOfColumns(pTableMeta)) { + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= numOfCols) { groupTag = true; } - + + SSqlGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr; + if (pGroupExpr->columnInfo == NULL) { + pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); + } + if (groupTag) { if (!UTIL_TABLE_IS_SUPERTABLE(pTableMetaInfo)) { return invalidSqlErrMsg(pQueryInfo->msg, msg9); } -// int32_t relIndex = index.columnIndex; -// if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { -// relIndex -= tscGetNumOfColumns(pTableMeta); -// } + int32_t relIndex = index.columnIndex; + if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { + relIndex -= numOfCols; + } - pQueryInfo->groupbyExpr.columnInfo[i] = - (SColIndex){.colIndex = index.columnIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId}; // relIndex; - addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[i].colIndex, index.tableIndex); + SColIndex colIndex = { + .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, + }; + + taosArrayPush(pGroupExpr->columnInfo, &colIndex); + addRequiredTagColumn(pTableMetaInfo, &index); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type > TSDB_DATA_TYPE_BINARY) { @@ -2528,8 +2546,11 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* } tscColumnListInsert(pQueryInfo->colList, &index); - pQueryInfo->groupbyExpr.columnInfo[i] = - (SColIndex){.colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId}; // relIndex; + + SColIndex colIndex = { + .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId, + }; + taosArrayPush(pGroupExpr->columnInfo, &colIndex); pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC; if (i == 0 && pList->nExpr > 1) { @@ -3524,7 +3545,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac return TSDB_CODE_SUCCESS; } - SStringBuilder sb1 = {0}; + SStringBuilder sb1; taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); char db[TSDB_TABLE_ID_LEN] = {0}; @@ -3707,14 +3728,14 @@ static void doAddJoinTagsColumnsIntoTagList(SQueryInfo* pQueryInfo, SCondExpr* p getColumnIndexByName(&pCondExpr->pJoinExpr->pLeft->colInfo, pQueryInfo, &index); pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - int32_t columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - addRequiredTagColumn(pQueryInfo, columnInfo, index.tableIndex); +// int32_t columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + addRequiredTagColumn(pTableMetaInfo, &index); getColumnIndexByName(&pCondExpr->pJoinExpr->pRight->colInfo, pQueryInfo, &index); pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - addRequiredTagColumn(pQueryInfo, columnInfo, index.tableIndex); +// columnInfo = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + addRequiredTagColumn(pTableMetaInfo, &index); } } @@ -4106,7 +4127,9 @@ int32_t parseOrderbyClause(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema if (index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { int32_t relTagIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (relTagIndex == pQueryInfo->groupbyExpr.columnInfo[0].colIndex) { + + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); + if (relTagIndex == pColIndex->colIndex) { orderByTags = true; } } else if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { @@ -4816,7 +4839,7 @@ void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t t if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) { SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE); + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false); pExpr->colInfo.flag = TSDB_COL_NORMAL; // NOTE: tag column does not add to source column list @@ -4845,7 +4868,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau int16_t bytes = pSchema[index.columnIndex].bytes; char* name = pSchema[index.columnIndex].name; - pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); + pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true); pExpr->colInfo.flag = TSDB_COL_TAG; // NOTE: tag column does not add to source column list @@ -4855,9 +4878,11 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau int32_t relIndex = index.columnIndex; pExpr->colInfo.colIndex = relIndex; - pQueryInfo->groupbyExpr.columnInfo[0].colIndex = relIndex; + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, 0); + pColIndex->colIndex = relIndex; - addRequiredTagColumn(pQueryInfo, pQueryInfo->groupbyExpr.columnInfo[0].colIndex, 0); + index = (SColumnIndex) {.tableIndex = tableIndex, .columnIndex = relIndex}; + addRequiredTagColumn(pTableMetaInfo, &index); } } } @@ -4870,15 +4895,17 @@ static void doLimitOutputNormalColOfGroupby(SSqlExpr* pExpr) { } void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { - int32_t index = pQueryInfo->groupbyExpr.columnInfo[tagIndex].colIndex; - + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, tagIndex); + int32_t index = pColIndex->colIndex; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index); SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = index}; size_t size = tscSqlExprNumOfExprs(pQueryInfo); - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_PRJ, &colIndex, pSchema->type, pSchema->bytes, pSchema->bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_PRJ, &colIndex, pSchema->type, pSchema->bytes, + pSchema->bytes, false); pExpr->colInfo.flag = TSDB_COL_NORMAL; doLimitOutputNormalColOfGroupby(pExpr); @@ -4930,7 +4957,9 @@ static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { if (pExpr->functionId == TSDB_FUNC_PRJ) { bool qualifiedCol = false; for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - if (pExpr->colInfo.colId == pQueryInfo->groupbyExpr.columnInfo[j].colId) { + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); + + if (pExpr->colInfo.colId == pColIndex->colId) { qualifiedCol = true; doLimitOutputNormalColOfGroupby(pExpr); pExpr->numOfParams = 1; @@ -4945,7 +4974,9 @@ static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) { static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) { for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - if (columnId == pGroupbyExpr->columnInfo[j].colId && pGroupbyExpr->columnInfo[j].flag == TSDB_COL_TAG) { + SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, j); + + if (columnId == pColIndex->colId && pColIndex->flag == TSDB_COL_TAG) { return true; } } @@ -5103,26 +5134,32 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { char* name = NULL; for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { - SColIndex* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[i]; - + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); + int16_t colIndex = pColIndex->colIndex; - if (pColIndex->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { type = TSDB_DATA_TYPE_BINARY; bytes = TSDB_TABLE_NAME_LEN; name = TSQL_TBNAME_L; } else { -// colIndex = (TSDB_COL_IS_TAG(pColIndex->flag)) ? tscGetNumOfColumns(pTableMetaInfo->pTableMeta) + pColIndex->colIndex -// : pColIndex->colIndex; - type = pSchema[colIndex].type; - bytes = pSchema[colIndex].bytes; - name = pSchema[colIndex].name; + if (TSDB_COL_IS_TAG(pColIndex->flag)) { + SSchema* tagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + + type = tagSchema[colIndex].type; + bytes = tagSchema[colIndex].bytes; + name = tagSchema[colIndex].name; + } else { + type = pSchema[colIndex].type; + bytes = pSchema[colIndex].bytes; + name = pSchema[colIndex].name; + } } size_t size = tscSqlExprNumOfExprs(pQueryInfo); if (TSDB_COL_IS_TAG(pColIndex->flag)) { SColumnIndex index = {.tableIndex = pQueryInfo->groupbyExpr.tableIndex, .columnIndex = colIndex}; - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG, &index, type, bytes, bytes, true); memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN); @@ -5131,7 +5168,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SQueryInfo* pQueryInfo) { // NOTE: tag column does not add to source column list SColumnList ids = getColumnList(1, 0, pColIndex->colIndex); - insertResultField(pQueryInfo, size - 1, &ids, bytes, type, name, pExpr); + insertResultField(pQueryInfo, size, &ids, bytes, type, name, pExpr); } else { // if this query is "group by" normal column, interval is not allowed if (pQueryInfo->intervalTime > 0) { @@ -5194,7 +5231,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { if (functId == TSDB_FUNC_PRJ && pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { bool qualified = false; for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { - SColIndex* pColIndex = &pQueryInfo->groupbyExpr.columnInfo[j]; + SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); if (pColIndex->colId == pExpr->colInfo.colId) { qualified = true; break; @@ -5272,7 +5309,7 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { SColumnIndex ind = {0}; SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false); const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name; strncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName)); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index bab66bcc18..5ae8f2402b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -642,8 +642,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pStart; - int32_t msgLen = 0; + int32_t msgLen = 0; int32_t numOfTables = 0; + int32_t numOfTags = taosArrayGetSize(pTableMetaInfo->tagColList); if (UTIL_TABLE_IS_NOMRAL_TABLE(pTableMetaInfo)) { numOfTables = 1; @@ -652,7 +653,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscTrace("%p queried tables:%d, table id: %s", pSql, 1, pTableMetaInfo->name); } else { // query super table int32_t index = pTableMetaInfo->vgroupIndex; - if (index < 0) { tscError("%p error vgroupIndex:%d", pSql, index); return -1; @@ -693,8 +693,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime); pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - - pQueryMsg->queryType = htons(pQueryInfo->type); + pQueryMsg->numOfTags = htonl(numOfTags); + + pQueryMsg->queryType = htons(pQueryInfo->type); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); pQueryMsg->numOfOutput = htons(numOfOutput); @@ -708,12 +709,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char *pMsg = (char *)(pQueryMsg->colList) + numOfCols * sizeof(SColumnInfo); SSchema *pSchema = tscGetTableSchema(pTableMeta); - int32_t total = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); for (int32_t i = 0; i < numOfCols; ++i) { SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; - if (pCol->colIndex.columnIndex >= total || pColSchema->type < TSDB_DATA_TYPE_BOOL || + if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR) { tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex, @@ -766,7 +766,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { - /* column id is not valid according to the cached metermeta, the table meta is expired */ + /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); return -1; } @@ -822,8 +822,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->orderType = htons(pGroupbyExpr->orderType); for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndex *pCol = &pGroupbyExpr->columnInfo[j]; - + SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); + *((int16_t *)pMsg) = pCol->colId; pMsg += sizeof(pCol->colId); @@ -844,6 +844,37 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(pQueryInfo->defaultVal[0]); } } + + if (numOfTags != 0) { + int32_t numOfColumns = tscGetNumOfColumns(pTableMeta); + int32_t numOfTagColumns = tscGetNumOfTags(pTableMeta); + int32_t total = numOfTagColumns + numOfColumns; + + pSchema = tscGetTableTagSchema(pTableMeta); + + for (int32_t i = 0; i < numOfTags; ++i) { + SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); + SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; + + if ((pCol->colIndex.columnIndex >= numOfTagColumns || pCol->colIndex.columnIndex < -1) || + (pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR)) { + tscError("%p sid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s", + pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, total, numOfTagColumns, + pCol->colIndex, pColSchema->name); + + return TSDB_CODE_INVALID_SQL; + } + + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + + pTagCol->colId = htons(pColSchema->colId); + pTagCol->bytes = htons(pColSchema->bytes); + pTagCol->type = htons(pColSchema->type); + pTagCol->numOfFilters = 0; + + pMsg += sizeof(SColumnInfo); + } + } // compressed ts block pQueryMsg->tsOffset = htonl(pMsg - pStart); @@ -2200,7 +2231,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { SFieldSupInfo* pInfo = tscFieldInfoAppend(pFieldInfo, &f); pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, - pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes); + pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes, false); } pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; @@ -2543,7 +2574,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { STableMetaInfo *pMMInfo = tscGetMetaInfo(pQueryInfo, i); STableMeta *pTableMeta = taosCacheAcquireByName(tscCacheHandle, pMMInfo->name); - tscAddTableMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); + tscAddTableMetaInfo(pNewQueryInfo, pMMInfo->name, pTableMeta, NULL, pMMInfo->tagColList); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 18a39acc0f..abdab2d9a1 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -947,7 +947,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { } static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize) { + int16_t size, int16_t interSize, bool isTagCol) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); @@ -958,22 +958,19 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; } else { - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); - pExpr->colInfo.colId = pSchema->colId; - } - - // tag columns require the column index revised. - int16_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - if (pColIndex->columnIndex >= numOfCols) { - pExpr->colInfo.flag = TSDB_COL_TAG; - } else { - if (pColIndex->columnIndex != TSDB_TBNAME_COLUMN_INDEX) { - pExpr->colInfo.flag = TSDB_COL_NORMAL; + if (isTagCol) { + SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId; + strncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, TSDB_COL_NAME_LEN); } else { - pExpr->colInfo.flag = TSDB_COL_TAG; + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); + pExpr->colInfo.colId = pSchema->colId; + strncpy(pExpr->colInfo.name, pSchema->name, TSDB_COL_NAME_LEN); } } + pExpr->colInfo.flag = isTagCol? TSDB_COL_TAG:TSDB_COL_NORMAL; + pExpr->colInfo.colIndex = pColIndex->columnIndex; pExpr->resType = type; pExpr->resBytes = size; @@ -984,20 +981,20 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol } SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize) { + int16_t size, int16_t interSize, bool isTagCol) { int32_t num = taosArrayGetSize(pQueryInfo->exprsInfo); if (index == num) { - return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, interSize); + return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol); } - SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize); + SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol); taosArrayInsert(pQueryInfo->exprsInfo, index, &pExpr); return pExpr; } SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize) { - SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize); + int16_t size, int16_t interSize, bool isTagCol) { + SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, interSize, isTagCol); taosArrayPush(pQueryInfo->exprsInfo, &pExpr); return pExpr; } @@ -1431,9 +1428,9 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); - int16_t actualTagIndex = pTableMetaInfo->tagColumnIndex[pExpr->colInfo.colIndex]; - - pColInfo[i].type = (actualTagIndex != -1) ? pTagSchema[actualTagIndex].type : TSDB_DATA_TYPE_BINARY; + + int16_t index = pExpr->colInfo.colIndex; + pColInfo[i].type = (index != -1) ? pTagSchema[index].type : TSDB_DATA_TYPE_BINARY; } else { pColInfo[i].type = pSchema[pExpr->colInfo.colIndex].type; } @@ -1645,7 +1642,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { } STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, int16_t numOfTags, int16_t* tags) { + SVgroupsInfo* vgroupList, SArray* pTagCols) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { return NULL; @@ -1663,7 +1660,6 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST } pTableMetaInfo->pTableMeta = pTableMeta; - pTableMetaInfo->numOfTags = numOfTags; if (vgroupList != NULL) { assert(vgroupList->numOfVgroups == 1); // todo fix me @@ -1674,16 +1670,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } - if (tags != NULL) { - memcpy(pTableMetaInfo->tagColumnIndex, tags, sizeof(pTableMetaInfo->tagColumnIndex[0]) * numOfTags); + if (pTagCols == NULL) { + pTableMetaInfo->tagColList = taosArrayInit(4, sizeof(SColumnIndex)); + } else { + pTableMetaInfo->tagColList = taosArrayClone(pTagCols); } - + pQueryInfo->numOfTables += 1; return pTableMetaInfo; } STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { - return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, 0, NULL); + return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL); } void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFromCache) { @@ -1857,17 +1855,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (pPrevSql == NULL) { STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name); - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->numOfTags, - pTableMetaInfo->tagColumnIndex); + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); STableMeta* pPrevTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta); + SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; pPrevInfo->vgroupList = NULL; - - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->numOfTags, - pTableMetaInfo->tagColumnIndex); + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList); } assert(pFinalInfo->pTableMeta != NULL && pNewQueryInfo->numOfTables == 1); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 636d5b863b..d32dfad59c 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -479,6 +479,7 @@ typedef struct { int32_t tsLen; // total length of ts comp block int32_t tsNumOfBlocks; // ts comp block numbers int32_t tsOrder; // ts comp block order + int32_t numOfTags; // number of tags columns involved SColumnInfo colList[]; } SQueryTableMsg; diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index a9a9424a7c..7887b25345 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -39,10 +39,10 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int typedef struct SSqlGroupbyExpr { int16_t tableIndex; + SArray* columnInfo; // SArray, group by columns information int16_t numOfGroupCols; - SColIndex* columnInfo; // group by columns information - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc + int16_t orderIndex; // order by column index + int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; typedef struct SPosInfo { @@ -116,6 +116,8 @@ typedef struct STableDataInfo { typedef struct SQuery { int16_t numOfCols; + int16_t numOfTags; + SOrderVal order; STimeWindow window; int64_t intervalTime; @@ -128,8 +130,9 @@ typedef struct SQuery { SLimitVal limit; int32_t rowSize; SSqlGroupbyExpr* pGroupbyExpr; - SArithExprInfo* pSelectExpr; + SArithExprInfo* pSelectExpr; SColumnInfo* colList; + SColumnInfo* tagColList; int32_t numOfFilterCols; int64_t* defaultVal; TSKEY lastKey; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index b2c814d101..df282b9845 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -289,7 +289,7 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { } for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); if (pColIndex->flag == TSDB_COL_NORMAL) { /* * make sure the normal column locates at the second position if tbname exists in group by clause @@ -312,7 +312,7 @@ int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { int16_t type = TSDB_DATA_TYPE_NULL; for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); if (pColIndex->flag == TSDB_COL_NORMAL) { colId = pColIndex->colId; break; @@ -996,12 +996,13 @@ static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int1 SSqlGroupbyExpr *pGroupbyExpr = pQuery->pGroupbyExpr; for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { - if (pGroupbyExpr->columnInfo[k].flag == TSDB_COL_TAG) { + SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); + if (pColIndex->flag == TSDB_COL_TAG) { continue; } int16_t colIndex = -1; - int32_t colId = pGroupbyExpr->columnInfo[k].colId; + int32_t colId = pColIndex->colId; for (int32_t i = 0; i < pQuery->numOfCols; ++i) { if (pQuery->colList[i].colId == colId) { @@ -1378,9 +1379,17 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->inputType = GET_COLUMN_TYPE(pQuery, i); - pCtx->inputBytes = GET_COLUMN_BYTES(pQuery, i); - + SColIndex* pIndex = &pSqlFuncMsg->colInfo; + + int32_t index = pSqlFuncMsg->colInfo.colIndex; + if (TSDB_COL_IS_TAG(pIndex->flag)) { + pCtx->inputBytes = pQuery->tagColList[index].bytes; + pCtx->inputType = pQuery->tagColList[index].type; + } else { + pCtx->inputBytes = pQuery->colList[index].bytes; + pCtx->inputType = pQuery->colList[index].type; + } + pCtx->ptsOutputBuf = NULL; pCtx->outputBytes = pQuery->pSelectExpr[i].bytes; @@ -4263,9 +4272,8 @@ static UNUSED_FUNC bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, STable } for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndex *pColIndex = &pGroupbyExpr->columnInfo[i]; + SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); if (pColIndex->flag == TSDB_COL_TAG) { - // assert(pSidset->numOfTables == pSidset->numOfSubSet); return true; } } @@ -5142,23 +5150,34 @@ static void stableQueryImpl(SQInfo *pQInfo) { sem_post(&pQInfo->dataReady); } -static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg) { +static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { int32_t j = 0; - - while (j < pQueryMsg->numOfCols) { - if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { - break; + + if (TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { + while(j < pQueryMsg->numOfTags) { + if (pExprMsg->colInfo.colId == pTagCols[j].colId) { + return j; + } + + j += 1; + } + + } else { + while (j < pQueryMsg->numOfCols) { + if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { + return j; + } + + j += 1; } - - j += 1; } - return j; + assert(0); } -bool vnodeValidateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg) { - int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg); - return j < pQueryMsg->numOfCols; +bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { + int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg, pTagCols); + return j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags; } static int32_t validateQueryMsg(SQueryTableMsg *pQueryMsg) { @@ -5228,7 +5247,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, - char **tagCond, SColIndex **groupbyCols) { + char **tagCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); @@ -5250,6 +5269,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); + pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); // query msg safety check if (validateQueryMsg(pQueryMsg) != 0) { @@ -5333,9 +5353,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, return TSDB_CODE_INVALID_QUERY_MSG; } } else { - if (!vnodeValidateExprColumnInfo(pQueryMsg, pExprMsg)) { - return TSDB_CODE_INVALID_QUERY_MSG; - } +// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { +// return TSDB_CODE_INVALID_QUERY_MSG; +// } } pExprMsg = (SSqlFuncMsg *)pMsg; @@ -5387,6 +5407,21 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pQueryMsg->tagCondLen > 0) { *tagCond = calloc(1, pQueryMsg->tagCondLen); memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); + pMsg += pQueryMsg->tagCondLen; + } + + if (pQueryMsg->numOfTags > 0) { + (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + + pTagCol->colId = htons(pTagCol->colId); + pTagCol->bytes = htons(pTagCol->bytes); + pTagCol->type = htons(pTagCol->type); + pTagCol->numOfFilters = 0; + + (*tagCols)[i] = *pTagCol; + } } qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 @@ -5452,7 +5487,7 @@ static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pExpr, SQueryTableMsg } static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExprInfo **pSqlFuncExpr, - SSqlFuncMsg **pExprMsg) { + SSqlFuncMsg **pExprMsg, SColumnInfo* pTagCols) { *pSqlFuncExpr = NULL; int32_t code = TSDB_CODE_SUCCESS; @@ -5483,10 +5518,10 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExp type = TSDB_DATA_TYPE_DOUBLE; bytes = tDataTypeDesc[type].nSize; } else { // parse the normal column - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase, pTagCols); assert(j < pQueryMsg->numOfCols); - SColumnInfo *pCol = &pQueryMsg->colList[j]; + SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].pBase.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j]; type = pCol->type; bytes = pCol->bytes; } @@ -5510,16 +5545,15 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExp for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { pExprs[i].pBase = *pExprMsg[i]; int16_t functId = pExprs[i].pBase.functionId; + if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase); + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase, pTagCols); assert(j < pQueryMsg->numOfCols); SColumnInfo *pCol = &pQueryMsg->colList[j]; - int16_t type = pCol->type; - int16_t bytes = pCol->bytes; int32_t ret = - getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, pExprs[i].pBase.arg[0].argValue.i64, + getResultDataInfo(pCol->type, pCol->bytes, functId, pExprs[i].pBase.arg[0].argValue.i64, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, tagLen, isSuperTable); assert(ret == TSDB_CODE_SUCCESS); } @@ -5547,7 +5581,11 @@ static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SCol pGroupbyExpr->orderType = pQueryMsg->orderType; pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; - pGroupbyExpr->columnInfo = pColIndex; + pGroupbyExpr->columnInfo = taosArrayInit(pQueryMsg->numOfGroupCols, sizeof(SColIndex)); + for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { + taosArrayPush(pGroupbyExpr->columnInfo, &pColIndex[i]); + } + return pGroupbyExpr; } @@ -5646,17 +5684,26 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } SColIndex *pColIndexEx = &pSqlExprMsg->colInfo; - for (int32_t f = 0; f < pQuery->numOfCols; ++f) { - if (pColIndexEx->colId == pQuery->colList[f].colId) { - pColIndexEx->colIndex = f; - break; + if (!TSDB_COL_IS_TAG(pColIndexEx->flag)) { + for (int32_t f = 0; f < pQuery->numOfCols; ++f) { + if (pColIndexEx->colId == pQuery->colList[f].colId) { + pColIndexEx->colIndex = f; + break; + } + } + } else { + for (int32_t f = 0; f < pQuery->numOfTags; ++f) { + if (pColIndexEx->colId == pQuery->tagColList[f].colId) { + pColIndexEx->colIndex = f; + break; + } } } } } static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SArithExprInfo *pExprs, - STableGroupInfo *groupInfo) { + STableGroupInfo *groupInfo, SColumnInfo* pTagCols) { SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); if (pQInfo == NULL) { return NULL; @@ -5680,8 +5727,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->slidingTime = pQueryMsg->slidingTime; pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit; pQuery->interpoType = pQueryMsg->interpoType; + pQuery->numOfTags = pQueryMsg->numOfTags; - pQuery->colList = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfCols); + pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); if (pQuery->colList == NULL) { goto _cleanup; } @@ -5692,7 +5740,16 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou SColumnInfo *pColInfo = &pQuery->colList[i]; pColInfo->filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pColInfo->numOfFilters); } - + + pQuery->tagColList = calloc(pQueryMsg->numOfTags, sizeof(SColumnInfo)); + if (pQuery->tagColList == NULL) { + goto _cleanup; + } + + for(int16_t i = 0; i < pQuery->numOfTags; ++i) { + pQuery->tagColList[i] = pTagCols[i]; + } + // calculate the result row size for (int16_t col = 0; col < numOfOutput; ++col) { assert(pExprs[col].bytes > 0); @@ -5743,8 +5800,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->pos = -1; - pQuery->window.skey = pQueryMsg->window.skey; - pQuery->window.ekey = pQueryMsg->window.ekey; + pQuery->window = pQueryMsg->window; pQuery->lastKey = pQuery->window.skey; if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { @@ -5955,12 +6011,14 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) int32_t code = TSDB_CODE_SUCCESS; - char * tagCond = NULL; - SArray * pTableIdList = NULL; + char * tagCond = NULL; + SArray * pTableIdList = NULL; SSqlFuncMsg **pExprMsg = NULL; - SColIndex * pGroupColIndex = NULL; + SColIndex * pGroupColIndex = NULL; + SColumnInfo* pTagColumnInfo = NULL; - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &pGroupColIndex)) != TSDB_CODE_SUCCESS) { + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, + &pGroupColIndex, &pTagColumnInfo)) != TSDB_CODE_SUCCESS) { return code; } @@ -5977,7 +6035,7 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) } SArithExprInfo *pExprs = NULL; - if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg)) != TSDB_CODE_SUCCESS) { + if ((code = createSqlFunctionExprFromMsg(pQueryMsg, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) { goto _query_over; } @@ -6010,7 +6068,7 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) } } - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo); + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &groupInfo, pTagColumnInfo); if ((*pQInfo) == NULL) { code = TSDB_CODE_SERV_OUT_OF_MEMORY; } From a2f054014d19564d3436e7cac7b6640bd277781b Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 24 Apr 2020 15:52:18 +0800 Subject: [PATCH 26/63] [td-171] fix bugs in select * for super table. --- cmake/platform.inc | 2 +- src/inc/tsdb.h | 1 + src/query/src/queryExecutor.c | 29 ++++++++++++++++++++++++----- src/tsdb/src/tsdbMeta.c | 12 ++++++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/cmake/platform.inc b/cmake/platform.inc index 2e0e2d6af0..1772a83fb2 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -62,7 +62,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) - ADD_DEFINITIONS(-D_M_X64) + ADD_DEFINITIONS(-D_M_X64 -D_DEBUG_VIEW) MESSAGE(STATUS "The current platform is Linux 64-bit") ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) IF (TD_ARM) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 8d33e0bbf7..481e05c102 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -98,6 +98,7 @@ int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); void tsdbClearTableCfg(STableCfg *config); int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId id, int32_t col, int16_t *type, int16_t *bytes, char **val); +int32_t tsdbTableGetName(TsdbRepoT *repo, STableId id, char** name); int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg); int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId); diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index df282b9845..3ad81ff4fc 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -1383,8 +1383,13 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int32_t index = pSqlFuncMsg->colInfo.colIndex; if (TSDB_COL_IS_TAG(pIndex->flag)) { - pCtx->inputBytes = pQuery->tagColList[index].bytes; - pCtx->inputType = pQuery->tagColList[index].type; + if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { + pCtx->inputBytes = TSDB_TABLE_NAME_LEN; + pCtx->inputType = TSDB_DATA_TYPE_BINARY; + } else { + pCtx->inputBytes = pQuery->tagColList[index].bytes; + pCtx->inputType = pQuery->tagColList[index].type; + } } else { pCtx->inputBytes = pQuery->colList[index].bytes; pCtx->inputType = pQuery->colList[index].type; @@ -2503,8 +2508,19 @@ static void doSetTagValueInParam(void *tsdb, STableId id, int32_t tagColId, tVar int16_t bytes = 0; int16_t type = 0; - tsdbGetTableTagVal(tsdb, id, tagColId, &type, &bytes, &val); + if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { + tsdbTableGetName(tsdb, id, &val); + bytes = TSDB_TABLE_NAME_LEN; + type = TSDB_DATA_TYPE_BINARY; + } else { + tsdbGetTableTagVal(tsdb, id, tagColId, &type, &bytes, &val); + } + tVariantCreateFromBinary(param, val, bytes, type); + + if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { + tfree(val); + } } void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void *tsdb) { @@ -5515,9 +5531,12 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExp return code; } - type = TSDB_DATA_TYPE_DOUBLE; + type = TSDB_DATA_TYPE_DOUBLE; bytes = tDataTypeDesc[type].nSize; - } else { // parse the normal column + } else if (pExprs[i].pBase.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { // parse the normal column + type = TSDB_DATA_TYPE_BINARY; + bytes = TSDB_TABLE_NAME_LEN; + } else{ int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase, pTagCols); assert(j < pQueryMsg->numOfCols); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 7c8d1fd51f..04863d1ef2 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -251,6 +251,18 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId id, int32_t colId, int16_t* return 0; } +int32_t tsdbTableGetName(TsdbRepoT *repo, STableId id, char** name) { + STsdbMeta* pMeta = tsdbGetMeta(repo); + STable* pTable = tsdbGetTableByUid(pMeta, id.uid); + + *name = strndup(pTable->name, TSDB_TABLE_NAME_LEN); + if (*name == NULL) { + return -1; + } else { + return 0; + } +} + int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { if (tsdbCheckTableCfg(pCfg) < 0) return -1; From 0eb45e7a2ee66ff378122e0ed27e0ee2ab33cbe5 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 24 Apr 2020 16:23:39 +0800 Subject: [PATCH 27/63] [TD-19] refactor http module --- src/plugins/http/src/gcHandle.c | 6 ++++-- src/plugins/http/src/gcJson.c | 6 ++---- src/plugins/http/src/httpAuth.c | 4 +++- src/plugins/http/src/httpCode.c | 3 +++ src/plugins/http/src/httpHandle.c | 17 +++++++---------- src/plugins/http/src/httpJson.c | 15 +++++---------- src/plugins/http/src/httpResp.c | 12 ++++++------ src/plugins/http/src/httpServer.c | 1 + src/plugins/http/src/httpSession.c | 16 +++++++--------- src/plugins/http/src/httpSql.c | 9 +++++---- src/plugins/http/src/httpSystem.c | 10 +++++----- src/plugins/http/src/httpUtil.c | 7 ++++--- src/plugins/http/src/restHandle.c | 4 +++- src/plugins/http/src/restJson.c | 1 + src/plugins/http/src/tgHandle.c | 13 +++++++------ src/plugins/http/src/tgJson.c | 5 +++-- 16 files changed, 66 insertions(+), 63 deletions(-) diff --git a/src/plugins/http/src/gcHandle.c b/src/plugins/http/src/gcHandle.c index 4980c9c804..fbe6757ccb 100644 --- a/src/plugins/http/src/gcHandle.c +++ b/src/plugins/http/src/gcHandle.c @@ -13,11 +13,13 @@ * along with this program. If not, see . */ -#include "gcHandle.h" +#define _DEFAULT_SOURCE +#include "os.h" #include "cJSON.h" +#include "httpLog.h" +#include "gcHandle.h" #include "gcJson.h" #include "taosdef.h" -#include "httpLog.h" static HttpDecodeMethod gcDecodeMethod = {"grafana", gcProcessRequest}; static HttpEncodeMethod gcHeartBeatMethod = {NULL, gcSendHeartBeatResp, NULL, NULL, NULL, NULL, NULL, NULL}; diff --git a/src/plugins/http/src/gcJson.c b/src/plugins/http/src/gcJson.c index 8f59633714..09990c123e 100644 --- a/src/plugins/http/src/gcJson.c +++ b/src/plugins/http/src/gcJson.c @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -#include -#include -#include - +#define _DEFAULT_SOURCE +#include "os.h" #include "gcHandle.h" #include "gcJson.h" #include "httpJson.h" diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c index adbf980f97..594f51e275 100644 --- a/src/plugins/http/src/httpAuth.c +++ b/src/plugins/http/src/httpAuth.c @@ -13,12 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" +#include "tkey.h" #include "tutil.h" #include "http.h" #include "httpLog.h" #include "httpHandle.h" -#include "tkey.h" + #define KEY_DES_4 4971256377704625728L bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) { diff --git a/src/plugins/http/src/httpCode.c b/src/plugins/http/src/httpCode.c index 230c0de2e0..9ec07fd851 100644 --- a/src/plugins/http/src/httpCode.c +++ b/src/plugins/http/src/httpCode.c @@ -13,6 +13,9 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + + char* httpMsg[] = { "success", // 0 "http server is not online", // 1 diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 529714c8e6..4eaff3660a 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -13,21 +13,18 @@ * along with this program. If not, see . */ -#include -#include -#include -#include - -#include "http.h" -#include "httpCode.h" -#include "httpHandle.h" -#include "httpResp.h" -#include "shash.h" +#define _DEFAULT_SOURCE +#include "os.h" #include "taos.h" #include "tglobal.h" #include "tsocket.h" #include "ttimer.h" +#include "shash.h" +#include "http.h" #include "httpLog.h" +#include "httpCode.h" +#include "httpHandle.h" +#include "httpResp.h" void httpToLowerUrl(char* url) { /*ignore case */ diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index fa74f857b3..cbca19b57f 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -13,20 +13,15 @@ * along with this program. If not, see . */ -#include -#include -#include -#include -#include -#include - +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosmsg.h" +#include "taoserror.h" #include "http.h" +#include "httpLog.h" #include "httpCode.h" #include "httpJson.h" #include "httpResp.h" -#include "taosmsg.h" -#include "httpLog.h" -#include "taoserror.h" #define MAX_NUM_STR_SZ 25 diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index c0a5c6d7ea..b064b58116 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -13,14 +13,14 @@ * along with this program. If not, see . */ -#include "httpResp.h" -#include -#include -#include "httpCode.h" -#include "httpJson.h" +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" #include "taosmsg.h" #include "httpLog.h" -#include "taoserror.h" +#include "httpResp.h" +#include "httpCode.h" +#include "httpJson.h" const char *httpKeepAliveStr[] = {"", "Connection: Keep-Alive\r\n", "Connection: Close\r\n"}; diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index f564863652..f36e0440cb 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" #include "tsocket.h" diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 39b2ff7d24..4b095aeb77 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -13,20 +13,18 @@ * along with this program. If not, see . */ -#include -#include -#include - -#include "http.h" -#include "httpCode.h" -#include "httpHandle.h" -#include "httpResp.h" - +#define _DEFAULT_SOURCE +#include "os.h" #include "shash.h" #include "taos.h" #include "ttime.h" #include "ttimer.h" +#include "http.h" #include "httpLog.h" +#include "httpCode.h" +#include "httpHandle.h" +#include "httpResp.h" + void httpAccessSession(HttpContext *pContext) { HttpServer *server = pContext->pThread->pServer; diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 017605093e..22ecdbc7b2 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" +#include "tnote.h" +#include "taos.h" +#include "tsclient.h" #include "http.h" +#include "httpLog.h" #include "httpCode.h" #include "httpHandle.h" #include "httpResp.h" -#include "taos.h" -#include "tsclient.h" -#include "tnote.h" -#include "httpLog.h" void *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos); diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 9afd5838c8..355fdf2fd0 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -15,17 +15,17 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "shash.h" +#include "taos.h" +#include "tglobal.h" +#include "tsocket.h" +#include "ttimer.h" #include "tadmin.h" #include "http.h" #include "httpCode.h" #include "httpHandle.h" #include "httpResp.h" #include "httpLog.h" -#include "shash.h" -#include "taos.h" -#include "tglobal.h" -#include "tsocket.h" -#include "ttimer.h" #include "gcHandle.h" #include "httpHandle.h" #include "restHandle.h" diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index c8d4737c41..fc5f771c69 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tmd5.h" +#include "shash.h" +#include "taos.h" #include "http.h" +#include "httpLog.h" #include "httpCode.h" #include "httpHandle.h" #include "httpResp.h" -#include "shash.h" -#include "taos.h" -#include "httpLog.h" bool httpCheckUsedbSql(char *sql) { if (strstr(sql, "use ") != NULL) { diff --git a/src/plugins/http/src/restHandle.c b/src/plugins/http/src/restHandle.c index 79d6fe5662..a2dc7d06a1 100644 --- a/src/plugins/http/src/restHandle.c +++ b/src/plugins/http/src/restHandle.c @@ -13,9 +13,11 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE +#include "os.h" +#include "httpLog.h" #include "restHandle.h" #include "restJson.h" -#include "httpLog.h" static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest}; static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest}; diff --git a/src/plugins/http/src/restJson.c b/src/plugins/http/src/restJson.c index 0489bcdbe0..9277d7d045 100644 --- a/src/plugins/http/src/restJson.c +++ b/src/plugins/http/src/restJson.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" #include "httpLog.h" diff --git a/src/plugins/http/src/tgHandle.c b/src/plugins/http/src/tgHandle.c index 5ea5f832d5..c854e0adc3 100644 --- a/src/plugins/http/src/tgHandle.c +++ b/src/plugins/http/src/tgHandle.c @@ -13,14 +13,15 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" -#include "tgHandle.h" -#include "shash.h" -#include "taosmsg.h" -#include "tgJson.h" -#include "taosdef.h" -#include "httpLog.h" #include "tglobal.h" +#include "shash.h" +#include "taosdef.h" +#include "taosmsg.h" +#include "tgHandle.h" +#include "tgJson.h" +#include "httpLog.h" /* * taos.telegraf.cfg formats like diff --git a/src/plugins/http/src/tgJson.c b/src/plugins/http/src/tgJson.c index f19e00fb98..1513d61d82 100644 --- a/src/plugins/http/src/tgJson.c +++ b/src/plugins/http/src/tgJson.c @@ -13,13 +13,14 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" +#include "taosmsg.h" +#include "httpLog.h" #include "httpJson.h" #include "httpResp.h" -#include "taosmsg.h" #include "tgHandle.h" #include "tgJson.h" -#include "httpLog.h" void tgInitQueryJson(HttpContext *pContext) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); From 7493384d661256482fcd4653fc30459e05530095 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 24 Apr 2020 16:56:36 +0800 Subject: [PATCH 28/63] [td-171] fix memory leaks in super table query. --- src/query/inc/queryExecutor.h | 2 +- src/query/src/queryExecutor.c | 67 ++++++++++++++++++++++------------- src/tsdb/src/tsdbRead.c | 5 +++ 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/query/inc/queryExecutor.h b/src/query/inc/queryExecutor.h index 7887b25345..13b0195893 100644 --- a/src/query/inc/queryExecutor.h +++ b/src/query/inc/queryExecutor.h @@ -108,7 +108,7 @@ typedef struct STableQueryInfo { SWindowResInfo windowResInfo; } STableQueryInfo; -typedef struct STableDataInfo { +typedef struct STableDataInfo { // todo merge with the STableQueryInfo struct int32_t tableIndex; int32_t groupIdx; // group id in table list STableQueryInfo* pTableQInfo; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 3ad81ff4fc..e2f3b2f655 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -111,7 +111,7 @@ static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size, int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); -static void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); +static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols); static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); static void createTableDataInfo(SQInfo *pQInfo); @@ -3478,7 +3478,7 @@ STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, int32_t tid return pTableQueryInfo; } -UNUSED_FUNC void destroyMeterQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) { +void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols) { if (pTableQueryInfo == NULL) { return; } @@ -4186,8 +4186,11 @@ int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, bool isSTableQuery) .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->groupInfo); + + if (!isSTableQuery || isIntervalQuery(pQuery) || isFixedOutputQuery(pQuery)) { + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->groupInfo); + } + pQInfo->tsdb = tsdb; pRuntimeEnv->pQuery = pQuery; @@ -4410,8 +4413,15 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1}; // include only current table + if (pRuntimeEnv->pQueryHandle != NULL) { + tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); + pRuntimeEnv->pQueryHandle = NULL; + } + pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp); - + taosArrayDestroy(tx); + taosArrayDestroy(g1); + if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->cur.vnodeIndex == -1) { int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; @@ -5760,14 +5770,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pColInfo->filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pColInfo->numOfFilters); } - pQuery->tagColList = calloc(pQueryMsg->numOfTags, sizeof(SColumnInfo)); - if (pQuery->tagColList == NULL) { - goto _cleanup; - } - - for(int16_t i = 0; i < pQuery->numOfTags; ++i) { - pQuery->tagColList[i] = pTagCols[i]; - } + pQuery->tagColList = pTagCols; // calculate the result row size for (int16_t col = 0; col < numOfOutput; ++col) { @@ -5930,10 +5933,6 @@ static void freeQInfo(SQInfo *pQInfo) { } } - tfree(pQuery->pFilterInfo); - tfree(pQuery->colList); - tfree(pQuery->sdata); - if (pQuery->pSelectExpr != NULL) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; @@ -5951,17 +5950,35 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->defaultVal); } - tfree(pQuery->pGroupbyExpr); - tfree(pQuery); - + // todo refactor, extract method to destroytableDataInfo int32_t numOfGroups = taosArrayGetSize(pQInfo->groupInfo.pGroupList); for (int32_t i = 0; i < numOfGroups; ++i) { SArray *p = taosArrayGetP(pQInfo->groupInfo.pGroupList, i); + + size_t num = taosArrayGetSize(p); + for(int32_t j = 0; j < num; ++j) { + SPair* pair = taosArrayGet(p, j); + if (pair->sec != NULL) { + destroyTableQueryInfo(((STableDataInfo*)pair->sec)->pTableQInfo, pQuery->numOfOutput); + tfree(pair->sec); + } + } taosArrayDestroy(p); } - + + if (pQuery->pGroupbyExpr != NULL) { + taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); + tfree(pQuery->pGroupbyExpr); + } + + tfree(pQuery->tagColList); + tfree(pQuery->pFilterInfo); + tfree(pQuery->colList); + tfree(pQuery->sdata); + taosArrayDestroy(pQInfo->groupInfo.pGroupList); - + tfree(pQuery); + qTrace("QInfo:%p QInfo is freed", pQInfo); // destroy signature, in order to avoid the query process pass the object safety check @@ -6036,8 +6053,8 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) SColIndex * pGroupColIndex = NULL; SColumnInfo* pTagColumnInfo = NULL; - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, - &pGroupColIndex, &pTagColumnInfo)) != TSDB_CODE_SUCCESS) { + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &pGroupColIndex, &pTagColumnInfo)) != + TSDB_CODE_SUCCESS) { return code; } @@ -6096,7 +6113,7 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) _query_over: taosArrayDestroy(pTableIdList); - + // if failed to add ref for all meters in this query, abort current query // atomic_fetch_add_32(&vnodeSelectReqNum, 1); return TSDB_CODE_SUCCESS; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 531a9e3b88..1c0c60f436 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1133,6 +1133,7 @@ static int32_t getAllTableIdList(STsdbRepo* tsdb, int64_t uid, SArray* list) { taosArrayPush(list, &t); } + tSkipListDestroyIter(iter); return TSDB_CODE_SUCCESS; } @@ -1445,6 +1446,7 @@ int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, pGroupInfo->numOfTables = taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + taosArrayDestroy(res); return ret; } @@ -1463,6 +1465,7 @@ int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, pGroupInfo->numOfTables = taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + taosArrayDestroy(res); return ret; } @@ -1483,6 +1486,7 @@ int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, int64_t uid, STableGroupInfo* pGro return TSDB_CODE_SUCCESS; } + void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; if (pQueryHandle == NULL) { @@ -1516,5 +1520,6 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tfree(pQueryHandle->pDataBlockInfo); tsdbDestroyHelper(&pQueryHandle->rhelper); + tfree(pQueryHandle); } From 102cf898a41e7f86937e05dd6bc39fb070052fa3 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 24 Apr 2020 16:59:21 +0800 Subject: [PATCH 29/63] [td-171] fix memory leaks in super table query. --- cmake/platform.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/platform.inc b/cmake/platform.inc index 1772a83fb2..2e0e2d6af0 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -62,7 +62,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) - ADD_DEFINITIONS(-D_M_X64 -D_DEBUG_VIEW) + ADD_DEFINITIONS(-D_M_X64) MESSAGE(STATUS "The current platform is Linux 64-bit") ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) IF (TD_ARM) From de50a2d704e5dacd42195e31011e7a8f0b729d25 Mon Sep 17 00:00:00 2001 From: localvar Date: Tue, 21 Apr 2020 19:44:17 +0800 Subject: [PATCH 30/63] TD-97: parse tbname criteria --- src/client/src/tscServer.c | 10 ++- src/inc/taosdef.h | 17 +++-- src/inc/taosmsg.h | 1 + src/inc/tsdb.h | 13 +++- src/query/inc/qast.h | 5 +- src/query/src/qast.c | 139 +++++++++++++++++++++++++++++----- src/query/src/queryExecutor.c | 15 +++- src/tsdb/src/tsdbRead.c | 64 ++++++++++------ src/util/inc/tarray.h | 17 ++++- src/util/inc/tbuffer.h | 2 +- src/util/src/tarray.c | 20 ++++- 11 files changed, 239 insertions(+), 64 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 15abcebe58..aeb137189b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -696,7 +696,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->numOfTags = htonl(numOfTags); - + pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); pQueryMsg->queryType = htons(pQueryInfo->type); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); @@ -915,6 +915,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } + if (pQueryInfo->tagCond.tbnameCond.cond == NULL) { + *pMsg = 0; + pMsg++; + } else { + strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond); + pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1; + } + // tbname in/like query expression should be sent to mgmt node msgLen = pMsg - pStart; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 1f64a0d5c1..b02a3df844 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -141,16 +141,17 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 +#define TSDB_RELATION_IN 8 -#define TSDB_RELATION_AND 8 -#define TSDB_RELATION_OR 9 -#define TSDB_RELATION_NOT 10 +#define TSDB_RELATION_AND 9 +#define TSDB_RELATION_OR 10 +#define TSDB_RELATION_NOT 11 -#define TSDB_BINARY_OP_ADD 11 -#define TSDB_BINARY_OP_SUBTRACT 12 -#define TSDB_BINARY_OP_MULTIPLY 13 -#define TSDB_BINARY_OP_DIVIDE 14 -#define TSDB_BINARY_OP_REMAINDER 15 +#define TSDB_BINARY_OP_ADD 12 +#define TSDB_BINARY_OP_SUBTRACT 13 +#define TSDB_BINARY_OP_MULTIPLY 14 +#define TSDB_BINARY_OP_DIVIDE 15 +#define TSDB_BINARY_OP_REMAINDER 16 #define TSDB_USERID_LEN 9 #define TS_PATH_DELIMITER_LEN 1 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index dd6dc52242..a72253d846 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -470,6 +470,7 @@ typedef struct { int64_t offset; uint16_t queryType; // denote another query process int16_t numOfOutput; // final output columns numbers + int16_t tagNameRelType; // relation of tag criteria and tbname criteria int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 481e05c102..e5c884c819 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -279,8 +279,17 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle); * @param pTagCond. tag query condition * */ -int32_t tsdbQueryByTagsCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, STableGroupInfo *pGroupList, - SColIndex *pColIndex, int32_t numOfCols); +int32_t tsdbQueryByTagsCond( + TsdbRepoT *tsdb, + int64_t uid, + const char *pTagCond, + size_t len, + int16_t tagNameRelType, + const char* tbnameCond, + STableGroupInfo *pGroupList, + SColIndex *pColIndex, + int32_t numOfCols + ); int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo); diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index f3484509f8..e1c61fceb1 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -35,6 +35,7 @@ enum { TSQL_NODE_EXPR = 0x1, TSQL_NODE_COL = 0x2, TSQL_NODE_VALUE = 0x4, + TSQL_NODE_ARRAY = 0x8, }; typedef bool (*__result_filter_fn_t)(const void *, void *); @@ -70,6 +71,7 @@ typedef struct tExprNode { struct tExprNode *pRight; // right child pointer } _node; struct SSchema *pSchema; + SArray* array; tVariant * pVal; }; } tExprNode; @@ -91,7 +93,8 @@ uint8_t getBinaryExprOptr(SSQLToken *pToken); SBuffer exprTreeToBinary(tExprNode* pExprTree); -int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode); +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size); +tExprNode* exprTreeFromTableName(const char* tbnameCond); #ifdef __cplusplus } diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 14f340b3d3..edfb78f0e4 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -460,6 +460,9 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { free((*pExpr)->pVal); } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { free((*pExpr)->pSchema); + } else if ((*pExpr)->nodeType == TSQL_NODE_ARRAY) { + // BUGBUG: memory leak, need free elements in the array + taosArrayDestroy((*pExpr)->array); } free(*pExpr); @@ -620,6 +623,8 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty // DEFAULT_COMP(p1, p2); //} +// develop_old mgmtSTableQuery for merge & intersect + int32_t merge(SArray *pLeft, SArray *pRight, SArray *pFinalRes) { // assert(pFinalRes->pRes == 0); // @@ -858,7 +863,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S tExprTreeTraverse(pSecond, NULL, result, param); } } else { // column project - assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); + assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_ARRAY)); param->setupInfoFn(pExpr, param->pExtInfo); if (pSkipList == NULL) { @@ -1032,49 +1037,141 @@ SBuffer exprTreeToBinary(tExprNode* pExprTree) { return buf; } -static void exprTreeFromBinaryImpl(tExprNode** pExprTree, SBuffer* pBuf) { +static tExprNode* exprTreeFromBinaryImpl(SBuffer* pBuf) { tExprNode* pExpr = calloc(1, sizeof(tExprNode)); - tbufReadToBuffer(pBuf, &pExpr->nodeType, sizeof(pExpr->nodeType)); + pExpr->nodeType = tbufReadUint8(pBuf); if (pExpr->nodeType == TSQL_NODE_VALUE) { tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + pExpr->pVal = pVal; - tbufReadToBuffer(pBuf, &pVal->nType, sizeof(pVal->nType)); + pVal->nType = tbufReadUint32(pBuf); if (pVal->nType == TSDB_DATA_TYPE_BINARY) { tbufReadToBuffer(pBuf, &pVal->nLen, sizeof(pVal->nLen)); pVal->pz = calloc(1, pVal->nLen + 1); tbufReadToBuffer(pBuf, pVal->pz, pVal->nLen); } else { - tbufReadToBuffer(pBuf, &pVal->pz, sizeof(pVal->i64Key)); + pVal->i64Key = tbufReadInt64(pBuf); } - pExpr->pVal = pVal; } else if (pExpr->nodeType == TSQL_NODE_COL) { SSchema* pSchema = calloc(1, sizeof(SSchema)); - tbufReadToBuffer(pBuf, &pSchema->colId, sizeof(pSchema->colId)); - tbufReadToBuffer(pBuf, &pSchema->bytes, sizeof(pSchema->bytes)); - tbufReadToBuffer(pBuf, &pSchema->type, sizeof(pSchema->type)); - + if (pSchema == NULL) { + // TODO: + } + pExpr->pSchema = pSchema; + + pSchema->colId = tbufReadInt16(pBuf); + pSchema->bytes = tbufReadInt16(pBuf); + pSchema->type = tbufReadUint8(pBuf); tbufReadToString(pBuf, pSchema->name, TSDB_COL_NAME_LEN); - pExpr->pSchema = pSchema; } else if (pExpr->nodeType == TSQL_NODE_EXPR) { - tbufReadToBuffer(pBuf, &pExpr->_node.optr, sizeof(pExpr->_node.optr)); - tbufReadToBuffer(pBuf, &pExpr->_node.hasPK, sizeof(pExpr->_node.hasPK)); - - exprTreeFromBinaryImpl(&pExpr->_node.pLeft, pBuf); - exprTreeFromBinaryImpl(&pExpr->_node.pRight, pBuf); + pExpr->_node.optr = tbufReadUint8(pBuf); + pExpr->_node.hasPK = tbufReadUint8(pBuf); + pExpr->_node.pLeft = exprTreeFromBinaryImpl(pBuf); + pExpr->_node.pRight = exprTreeFromBinaryImpl(pBuf); assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL); } - *pExprTree = pExpr; + return pExpr; } -int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode) { +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) { + if (size == 0) { + return NULL; + } SBuffer rbuf = {0}; - /*int32_t code =*/ tbufBeginRead(&rbuf, pBuf, size); - exprTreeFromBinaryImpl(pExprNode, &rbuf); - return TSDB_CODE_SUCCESS; + tbufBeginRead(&rbuf, pBuf, size); + return exprTreeFromBinaryImpl(&rbuf); } +static int cmpStrInArray(const void* a, const void* b) { + const char* x = *(const char**)a; + const char* y = *(const char**)b; + return strcmp(x, y); +} + +tExprNode* exprTreeFromTableName(const char* tbnameCond) { + if (!tbnameCond) { + return NULL; + } + + tExprNode* expr = calloc(1, sizeof(tExprNode)); + if (expr == NULL) { + // TODO: + } + expr->nodeType = TSQL_NODE_EXPR; + + tExprNode* left = calloc(1, sizeof(tExprNode)); + if (left == NULL) { + // TODO: + } + expr->_node.pLeft = left; + + left->nodeType = TSQL_NODE_COL; + SSchema* pSchema = calloc(1, sizeof(SSchema)); + if (pSchema == NULL) { + // TODO: + } + left->pSchema = pSchema; + + pSchema->type = TSDB_DATA_TYPE_BINARY; + pSchema->bytes = TSDB_TABLE_NAME_LEN; + strcpy(pSchema->name, TSQL_TBNAME_L); + pSchema->colId = -1; + + tExprNode* right = calloc(1, sizeof(tExprNode)); + if (right == NULL) { + // TODO + } + expr->_node.pRight = right; + + if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) { + expr->_node.optr = TSDB_RELATION_LIKE; + tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + right->pVal = pVal; + pVal->nType = TSDB_DATA_TYPE_BINARY; + size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1; + pVal->pz = malloc(len); + if (pVal->pz == NULL) { + // TODO: + } + memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN, len); + + } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { + expr->_node.optr = TSDB_RELATION_IN; + right->array = taosArrayInit(2, sizeof(char*)); + + const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN; + for (const char *e = cond; *e != 0; e++) { + if (*e == ',') { + size_t len = e - cond + 1; + char* p = malloc( len ); + memcpy(p, cond, len); + p[len - 1] = 0; + cond += len; + taosArrayPush(right->array, &p); + } + } + + if (*cond != 0) { + size_t len = strlen(cond) + 1; + char* p = malloc( len ); + memcpy(p, cond, len); + p[len - 1] = 0; + taosArrayPush(right->array, &p); + } + + taosArraySort(right->array, cmpStrInArray); + } + + return expr; +} \ No newline at end of file diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index e2f3b2f655..cb3bf86ebe 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5286,6 +5286,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->order = htons(pQueryMsg->order); pQueryMsg->orderColId = htons(pQueryMsg->orderColId); pQueryMsg->queryType = htons(pQueryMsg->queryType); + pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); @@ -5450,6 +5451,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } } + if (*pMsg != 0) { + size_t len = strlen(pMsg); + *tbnameCond = malloc(len + 1); + strcpy(*tbnameCond, pMsg); + } + qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, ts order:%d, " "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 @@ -6047,13 +6054,13 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) int32_t code = TSDB_CODE_SUCCESS; - char * tagCond = NULL; + char * tagCond = NULL, *tbnameCond = NULL; SArray * pTableIdList = NULL; SSqlFuncMsg **pExprMsg = NULL; SColIndex * pGroupColIndex = NULL; SColumnInfo* pTagColumnInfo = NULL; - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &pGroupColIndex, &pTagColumnInfo)) != + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) != TSDB_CODE_SUCCESS) { return code; } @@ -6089,7 +6096,7 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) STableId *id = taosArrayGet(pTableIdList, 0); id->uid = -1; // todo fix me - /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, &groupInfo, pGroupColIndex, + /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, pQueryMsg->numOfGroupCols); if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query code = TSDB_CODE_SUCCESS; @@ -6112,6 +6119,8 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) code = initQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); _query_over: + tfree(tagCond); + tfree(tbnameCond); taosArrayDestroy(pTableIdList); // if failed to add ref for all meters in this query, abort current query diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 2357cc892f..60fa46607a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1327,12 +1327,9 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table - size_t num = taosArrayGetSize(pTableList); - - SArray* sa = taosArrayInit(num, sizeof(SPair)); - for(int32_t i = 0; i < num; ++i) { + SArray* sa = taosArrayInit(size, sizeof(SPair)); + for(int32_t i = 0; i < size; ++i) { STable* pTable = taosArrayGetP(pTableList, i); - SPair p = {.first = pTable}; taosArrayPush(sa, &p); } @@ -1393,6 +1390,9 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { case TSDB_RELATION_LIKE: { return ret == 0; } + case TSDB_RELATION_IN: { + + } default: assert(false); @@ -1400,6 +1400,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { return true; } + static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the binary expression STSchema* pSchema = pSTable->tagSchema; @@ -1422,12 +1423,22 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) tExprTreeDestroy(&pExpr, destroyHelper); convertQueryResult(pRes, pTableList); + taosArrayDestroy(pTableList); return TSDB_CODE_SUCCESS; } -int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols) { - + +int32_t tsdbQueryByTagsCond( + TsdbRepoT *tsdb, + int64_t uid, + const char *pTagCond, + size_t len, + int16_t tagNameRelType, + const char* tbnameCond, + STableGroupInfo *pGroupList, + SColIndex *pColIndex, + int32_t numOfCols +) { STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); if (pSTable == NULL) { uError("failed to get stable, uid:%" PRIu64, uid); @@ -1437,11 +1448,12 @@ int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, SArray* res = taosArrayInit(8, POINTER_BYTES); STSchema* pTagSchema = tsdbGetTableTagSchema(tsdbGetMeta(tsdb), pSTable); - if (pTagCond == NULL || len == 0) { // no tags condition, all tables created according to this stable are involved + // no tags and tbname condition, all child tables of this stable are involved + if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { int32_t ret = getAllTableIdList(tsdb, uid, res); - if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(res); - return ret; + if (ret == TSDB_CODE_SUCCESS) { + pGroupList->numOfTables = taosArrayGetSize(res); + pGroupList->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); } pGroupInfo->numOfTables = taosArrayGetSize(res); @@ -1450,20 +1462,24 @@ int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, return ret; } - tExprNode* pExprNode = NULL; - int32_t ret = TSDB_CODE_SUCCESS; - - // failed to build expression, no result, return immediately - if ((ret = exprTreeFromBinary(pTagCond, len, &pExprNode) != TSDB_CODE_SUCCESS) || (pExprNode == NULL)) { - uError("stable:%" PRIu64 ", failed to deserialize expression tree, error exists", uid); - taosArrayDestroy(res); - return ret; + tExprNode* expr = exprTreeFromTableName(tbnameCond); + tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); + if (tagExpr != NULL) { + if (expr == NULL) { + expr = tagExpr; + } else { + tExprNode* tbnameExpr = expr; + expr = calloc(1, sizeof(tExprNode)); + expr->nodeType = TSQL_NODE_EXPR; + expr->_node.optr = tagNameRelType; + expr->_node.pLeft = tbnameExpr; + expr->_node.pRight = tagExpr; + } } - doQueryTableList(pSTable, res, pExprNode); - - pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + doQueryTableList(pSTable, res, expr); + pGroupList->numOfTables = taosArrayGetSize(res); + pGroupList->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); taosArrayDestroy(res); return ret; diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 6fab24d51a..bd601a743f 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -53,7 +53,7 @@ void* taosArrayPush(SArray* pArray, void* pData); * * @param pArray */ -void taosArrayPop(SArray* pArray); +void* taosArrayPop(SArray* pArray); /** * get the data from array @@ -112,6 +112,21 @@ SArray* taosArrayClone(SArray* pSrc); */ void taosArrayDestroy(SArray* pArray); +/** + * sort the array + * @param pArray + * @param compar + */ +void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)); + +/** + * search the array + * @param pArray + * @param compar + * @param key + */ +void* taosArraySearch(SArray* pArray, int (*compar)(const void*, const void*), const void* key); + #ifdef __cplusplus } #endif diff --git a/src/util/inc/tbuffer.h b/src/util/inc/tbuffer.h index d52031fc6a..9bc0fd9f43 100644 --- a/src/util/inc/tbuffer.h +++ b/src/util/inc/tbuffer.h @@ -120,7 +120,7 @@ void tbufWriteString(SBuffer* buf, const char* str); TBUFFER_DEFINE_FUNCTION(bool, Bool) TBUFFER_DEFINE_FUNCTION(char, Char) TBUFFER_DEFINE_FUNCTION(int8_t, Int8) -TBUFFER_DEFINE_FUNCTION(uint8_t, Unt8) +TBUFFER_DEFINE_FUNCTION(uint8_t, Uint8) TBUFFER_DEFINE_FUNCTION(int16_t, Int16) TBUFFER_DEFINE_FUNCTION(uint16_t, Uint16) TBUFFER_DEFINE_FUNCTION(int32_t, Int32) diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index c025958438..39bbde3601 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -76,12 +76,13 @@ void* taosArrayPush(SArray* pArray, void* pData) { return dst; } -void taosArrayPop(SArray* pArray) { +void* taosArrayPop(SArray* pArray) { if (pArray == NULL || pArray->size == 0) { - return; + return NULL; } pArray->size -= 1; + return TARRAY_GET_ELEM(pArray, pArray->size); } void* taosArrayGet(const SArray* pArray, size_t index) { @@ -183,3 +184,18 @@ void taosArrayDestroy(SArray* pArray) { free(pArray->pData); free(pArray); } + +void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) { + assert(pArray != NULL); + assert(compar != NULL); + + qsort(pArray->pData, pArray->size, pArray->elemSize, compar); +} + +void* taosArraySearch(SArray* pArray, int (*compar)(const void*, const void*), const void* key) { + assert(pArray != NULL); + assert(compar != NULL); + assert(key != NULL); + + return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, compar); +} From 87363383f955df5aa5773bce8a13635a29be5e7d Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 23 Apr 2020 15:11:44 +0800 Subject: [PATCH 31/63] TD-97: search in array is done --- src/inc/taosdef.h | 14 +++ src/query/inc/qast.h | 2 - src/query/inc/tvariant.h | 2 + src/query/src/qast.c | 251 +++++++++++++++++++++++---------------- src/query/src/tvariant.c | 23 ++++ src/tsdb/src/tsdbRead.c | 4 +- src/util/inc/tarray.h | 15 ++- src/util/src/tarray.c | 26 +++- src/util/src/tcompare.c | 67 +++++------ 9 files changed, 260 insertions(+), 144 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index b02a3df844..15fb026381 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -32,6 +32,20 @@ extern "C" { #define TSKEY int64_t #endif +// Data type definition +#define TSDB_DATA_TYPE_NULL 0 // 1 bytes +#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes +#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte +#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes +#define TSDB_DATA_TYPE_INT 4 // 4 bytes +#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes +#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes +#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes +#define TSDB_DATA_TYPE_BINARY 8 // string +#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes +#define TSDB_DATA_TYPE_NCHAR 10 // unicode string +#define TSDB_DATA_TYPE_ARRAY 11 // only used 'in' query to hold the values + // Bytes for each type. extern const int32_t TYPE_BYTES[11]; // TODO: replace and remove code below diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index e1c61fceb1..7de1e5be66 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -35,7 +35,6 @@ enum { TSQL_NODE_EXPR = 0x1, TSQL_NODE_COL = 0x2, TSQL_NODE_VALUE = 0x4, - TSQL_NODE_ARRAY = 0x8, }; typedef bool (*__result_filter_fn_t)(const void *, void *); @@ -71,7 +70,6 @@ typedef struct tExprNode { struct tExprNode *pRight; // right child pointer } _node; struct SSchema *pSchema; - SArray* array; tVariant * pVal; }; } tExprNode; diff --git a/src/query/inc/tvariant.h b/src/query/inc/tvariant.h index 7ed5aa8dbe..9e11e6c294 100644 --- a/src/query/inc/tvariant.h +++ b/src/query/inc/tvariant.h @@ -17,6 +17,7 @@ #define TDENGINE_TVARIANT_H #include "tstoken.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { @@ -31,6 +32,7 @@ typedef struct tVariant { double dKey; char * pz; wchar_t *wpz; + SArray *arr; // only for 'in' query to hold value list, not value for a field }; } tVariant; diff --git a/src/query/src/qast.c b/src/query/src/qast.c index edfb78f0e4..4da26e0746 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -460,9 +460,6 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { free((*pExpr)->pVal); } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { free((*pExpr)->pSchema); - } else if ((*pExpr)->nodeType == TSQL_NODE_ARRAY) { - // BUGBUG: memory leak, need free elements in the array - taosArrayDestroy((*pExpr)->array); } free(*pExpr); @@ -524,38 +521,48 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { pCond->start = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->start->v, &queryColInfo->q); pCond->start->optr = queryColInfo->optr; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { pCond->end = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->end->v, &queryColInfo->q); pCond->end->optr = queryColInfo->optr; + + } else if (optr == TSDB_RELATION_IN) { + printf("relation is in\n"); + + } else if (optr == TSDB_RELATION_LIKE) { + printf("relation is like\n"); + } return TSDB_CODE_SUCCESS; } -static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t type, SArray* result) { +static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { SSkipListIterator* iter = NULL; - - if (pCond->start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->start->v.i64Key, type, TSDB_ORDER_ASC); + int32_t type = pQueryInfo->q.nType; + + SQueryCond cond = { 0 }; + setQueryCond(pQueryInfo, &cond); + + if (cond.start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.start->v.i64Key, type, TSDB_ORDER_ASC); } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->end->v.i64Key, type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.end->v.i64Key, type, TSDB_ORDER_DESC); } __compar_fn_t func = getComparFunc(pSkipList->keyInfo.type, type, 0); - if (pCond->start != NULL) { - int32_t optr = pCond->start->optr; + if (cond.start != NULL) { + int32_t optr = cond.start->optr; if (optr == TSDB_RELATION_EQUAL) { while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); if (ret == 0) { taosArrayPush(result, SL_GET_NODE_DATA(pNode)); } else { @@ -570,7 +577,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); assert(ret >= 0); } @@ -587,7 +594,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty assert(0); } } else { - int32_t optr = pCond->end->optr; + int32_t optr = cond.end->optr; if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { bool comp = true; @@ -597,7 +604,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->end->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.end->v.i64Key); assert(ret <= 0); } @@ -788,6 +795,36 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki } } + +#include + +static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + bool addToResult = false; + + SSkipListNode *pNode = tSkipListIterGet(iter); + STable* table = *(STable**) SL_GET_NODE_DATA(pNode); + + if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->optr == TSDB_RELATION_IN) { + addToResult = pQueryInfo->compare(table->name, pQueryInfo->q.arr); + } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) { + addToResult = pQueryInfo->compare(table->name, pQueryInfo->q.pz); + } + } else { + // TODO: other columns + } + + if (addToResult) { + taosArrayPush(result, table); + } + } +} + + + // post-root order traverse syntax tree void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { if (pExpr == NULL) { @@ -797,95 +834,99 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S tExprNode *pLeft = pExpr->_node.pLeft; tExprNode *pRight = pExpr->_node.pRight; - // recursive traverse left child branch - if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { - uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + // column project + if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { + assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); - if (weight == 0 && taosArrayGetSize(result) > 0 && pSkipList == NULL) { - /** - * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. - * Since no index presented, the filter operation is done by scan all elements in the result set. - * - * if the query is a high selectivity filter, only small portion of meters are retrieved. - */ + param->setupInfoFn(pExpr, param->pExtInfo); + if (pSkipList == NULL) { + tSQLListTraverseOnResult(pExpr, param->fp, result); + return; + } + + tQueryInfo *pQueryInfo = pExpr->_node.info; + if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { + tQueryIndexColumn(pSkipList, pQueryInfo, result); + } else { + tQueryIndexlessColumn(pSkipList, pQueryInfo, result); + } + + return; + } + + // recursive traverse left child branch + uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + + if (weight == 0 ) { + if (taosArrayGetSize(result) > 0 && pSkipList == NULL) { + /** + * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. + * Since no index presented, the filter operation is done by scan all elements in the result set. + * + * if the query is a high selectivity filter, only small portion of meters are retrieved. + */ exprTreeTraverseImpl(pExpr, result, param); - } else if (weight == 0) { + } else { /** * apply the hierarchical expression to every node in skiplist for find the qualified nodes */ assert(taosArrayGetSize(result) == 0); tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - } else if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { - SArray* rLeft = taosArrayInit(10, POINTER_BYTES); - SArray* rRight = taosArrayInit(10, POINTER_BYTES); - - tExprTreeTraverse(pLeft, pSkipList, rLeft, param); - tExprTreeTraverse(pRight, pSkipList, rRight, param); - - if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS - intersect(rLeft, rRight, result); - } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - merge(rLeft, rRight, result); - } else { - assert(false); - } - - taosArrayDestroy(rLeft); - taosArrayDestroy(rRight); - } else { - /* - * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here - * - * first, we filter results based on the skiplist index, which is the initial filter stage, - * then, we conduct the secondary filter operation based on the result from the initial filter stage. - */ - assert(pExpr->_node.optr == TSDB_RELATION_AND); - - tExprNode *pFirst = NULL; - tExprNode *pSecond = NULL; - if (pLeft->_node.hasPK == 1) { - pFirst = pLeft; - pSecond = pRight; - } else { - pFirst = pRight; - pSecond = pLeft; - } - - assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); - - // we filter the result based on the skiplist index in the first place - tExprTreeTraverse(pFirst, pSkipList, result, param); - - /* - * recursively perform the filter operation based on the initial results, - * So, we do not set the skip list index as a parameter - */ - tExprTreeTraverse(pSecond, NULL, result, param); - } - } else { // column project - assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_ARRAY)); - - param->setupInfoFn(pExpr, param->pExtInfo); - if (pSkipList == NULL) { - tSQLListTraverseOnResult(pExpr, param->fp, result); - } else { - tQueryInfo *pQueryInfo = pExpr->_node.info; - - if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { - SQueryCond cond = {0}; - /*int32_t ret = */ setQueryCond(pQueryInfo, &cond); - tQueryOnSkipList(pSkipList, &cond, pQueryInfo->q.nType, result); - } else { - /* Brutal force scan the whole skip list to find the appropriate result, - * since the filter is not applied to indexed column. - */ - assert(0); -// result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, queryColInfo); - } } + return; } + + if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { + SArray* rLeft = taosArrayInit(10, POINTER_BYTES); + SArray* rRight = taosArrayInit(10, POINTER_BYTES); + + tExprTreeTraverse(pLeft, pSkipList, rLeft, param); + tExprTreeTraverse(pRight, pSkipList, rRight, param); + + if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS + intersect(rLeft, rRight, result); + } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + merge(rLeft, rRight, result); + } else { + assert(false); + } + + taosArrayDestroy(rLeft); + taosArrayDestroy(rRight); + return; + } + + /* + * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here + * + * first, we filter results based on the skiplist index, which is the initial filter stage, + * then, we conduct the secondary filter operation based on the result from the initial filter stage. + */ + assert(pExpr->_node.optr == TSDB_RELATION_AND); + + tExprNode *pFirst = NULL; + tExprNode *pSecond = NULL; + if (pLeft->_node.hasPK == 1) { + pFirst = pLeft; + pSecond = pRight; + } else { + pFirst = pRight; + pSecond = pLeft; + } + + assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); + + // we filter the result based on the skiplist index in the first place + tExprTreeTraverse(pFirst, pSkipList, result, param); + + /* + * recursively perform the filter operation based on the initial results, + * So, we do not set the skip list index as a parameter + */ + tExprTreeTraverse(pSecond, NULL, result, param); } + void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, char *, int32_t)) { if (pExprs == NULL) { @@ -1090,12 +1131,6 @@ tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) { return exprTreeFromBinaryImpl(&rbuf); } -static int cmpStrInArray(const void* a, const void* b) { - const char* x = *(const char**)a; - const char* y = *(const char**)b; - return strcmp(x, y); -} - tExprNode* exprTreeFromTableName(const char* tbnameCond) { if (!tbnameCond) { return NULL; @@ -1132,6 +1167,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { expr->_node.pRight = right; if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) { + right->nodeType = TSQL_NODE_VALUE; expr->_node.optr = TSDB_RELATION_LIKE; tVariant* pVal = calloc(1, sizeof(tVariant)); if (pVal == NULL) { @@ -1147,8 +1183,15 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN, len); } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { + right->nodeType = TSQL_NODE_VALUE; expr->_node.optr = TSDB_RELATION_IN; - right->array = taosArrayInit(2, sizeof(char*)); + tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + right->pVal = pVal; + pVal->nType = TSDB_DATA_TYPE_ARRAY; + pVal->arr = taosArrayInit(2, sizeof(char*)); const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN; for (const char *e = cond; *e != 0; e++) { @@ -1158,7 +1201,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { memcpy(p, cond, len); p[len - 1] = 0; cond += len; - taosArrayPush(right->array, &p); + taosArrayPush(pVal->arr, &p); } } @@ -1167,10 +1210,10 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { char* p = malloc( len ); memcpy(p, cond, len); p[len - 1] = 0; - taosArrayPush(right->array, &p); + taosArrayPush(pVal->arr, &p); } - taosArraySort(right->array, cmpStrInArray); + taosArraySortString(pVal->arr); } return expr; diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index f0addb733b..e90cc3a44d 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -130,6 +130,17 @@ void tVariantDestroy(tVariant *pVar) { tfree(pVar->pz); pVar->nLen = 0; } + + // NOTE: this is only for string array + if (pVar->nType == TSDB_DATA_TYPE_ARRAY) { + size_t num = taosArrayGetSize(pVar->arr); + for(size_t i = 0; i < num; i++) { + void* p = taosArrayGetP(pVar->arr, i); + free(p); + } + taosArrayDestroy(pVar->arr); + pVar->arr = NULL; + } } void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { @@ -145,6 +156,18 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { pDst->pz = calloc(1, len); memcpy(pDst->pz, pSrc->pz, len); + return; + } + + // this is only for string array + if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { + size_t num = taosArrayGetSize(pSrc->arr); + pDst->arr = taosArrayInit(num, sizeof(char*)); + for(size_t i = 0; i < num; i++) { + char* p = (char*)taosArrayGetP(pSrc->arr, i); + char* n = strdup(p); + taosArrayPush(pDst->arr, &n); + } } } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 60fa46607a..58fd4eb653 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1213,7 +1213,9 @@ void filterPrepare(void* expr, void* param) { pInfo->compare = getComparFunc(pSchema->type, pCond->nType, pInfo->optr); tVariantAssign(&pInfo->q, pCond); - tVariantTypeSetType(&pInfo->q, pInfo->sch.type); + if (pInfo->optr != TSDB_RELATION_IN) { + tVariantTypeSetType(&pInfo->q, pInfo->sch.type); + } } int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index bd601a743f..dbb74edf07 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -119,13 +119,26 @@ void taosArrayDestroy(SArray* pArray); */ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)); +/** + * sort string array + * @param pArray + */ +void taosArraySortString(SArray* pArray); + /** * search the array * @param pArray * @param compar * @param key */ -void* taosArraySearch(SArray* pArray, int (*compar)(const void*, const void*), const void* key); +void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key); + +/** + * search the array + * @param pArray + * @param key + */ +char* taosArraySearchString(const SArray* pArray, const char* key); #ifdef __cplusplus } diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 39bbde3601..ef57c2432f 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -27,7 +27,7 @@ void* taosArrayInit(size_t size, size_t elemSize) { return NULL; } - pArray->pData = calloc(size, elemSize * size); + pArray->pData = calloc(size, elemSize); if (pArray->pData == NULL) { free(pArray); return NULL; @@ -192,10 +192,32 @@ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) { qsort(pArray->pData, pArray->size, pArray->elemSize, compar); } -void* taosArraySearch(SArray* pArray, int (*compar)(const void*, const void*), const void* key) { +void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key) { assert(pArray != NULL); assert(compar != NULL); assert(key != NULL); return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, compar); } + +static int taosArrayCompareString(const void* a, const void* b) { + const char* x = *(const char**)a; + const char* y = *(const char**)b; + return strcmp(x, y); +} + +void taosArraySortString(SArray* pArray) { + assert(pArray != NULL); + qsort(pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString); +} + +char* taosArraySearchString(const SArray* pArray, const char* key) { + assert(pArray != NULL); + assert(key != NULL); + + void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString); + if (p == NULL) { + return NULL; + } + return *(char**)p; +} \ No newline at end of file diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 4505ea533a..d4cfd983d5 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -1,42 +1,34 @@ #include "taosdef.h" #include "tcompare.h" - +#include #include "tutil.h" int32_t compareInt32Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int64_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt16Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT16_VAL(pLeft) - GET_INT16_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt8Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT8_VAL(pLeft) - GET_INT8_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { @@ -69,12 +61,7 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { } int32_t compareStrVal(const void *pLeft, const void *pRight) { - int32_t ret = strcmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + return (int32_t)strcmp(pLeft, pRight); } int32_t compareWStrVal(const void *pLeft, const void *pRight) { @@ -228,6 +215,11 @@ static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } +static int32_t compareStrInList(const void* pLeft, const void* pRight) { + const SArray* arr = (const SArray*)pRight; + return taosArraySearchString(arr, &pLeft) == NULL ? 0 : 1; +} + static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; @@ -250,7 +242,6 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { // assert(type == filterDataType); - if (filterDataType == TSDB_DATA_TYPE_BIGINT || filterDataType == TSDB_DATA_TYPE_TIMESTAMP) { comparFn = compareInt64Val; } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { @@ -259,6 +250,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) break; } + case TSDB_DATA_TYPE_BOOL: { if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { comparFn = compareInt32Val; @@ -267,6 +259,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } break; } + case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { @@ -276,12 +269,18 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } break; } + case TSDB_DATA_TYPE_BINARY: { - assert(filterDataType == TSDB_DATA_TYPE_BINARY); - if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrPatternComp; + + } else if (optr == TSDB_RELATION_IN) { + assert(filterDataType == TSDB_DATA_TYPE_ARRAY); + comparFn = compareStrInList; + } else { /* normal relational comparFn */ + assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrVal; } From 5741959c9642891403080b2680922069bb217a8f Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 23 Apr 2020 15:50:03 +0800 Subject: [PATCH 32/63] TD-97: tbname query complete --- src/query/src/qast.c | 4 ++-- src/util/src/tcompare.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 4da26e0746..7b97e6f9c5 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -30,6 +30,7 @@ #include "tarray.h" #include "tskiplist.h" #include "queryLog.h" +#include "tsdbMain.h" /* * @@ -796,7 +797,6 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki } -#include static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); @@ -818,7 +818,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, } if (addToResult) { - taosArrayPush(result, table); + taosArrayPush(result, (void*)&table); } } } diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index d4cfd983d5..98e0c9c9a2 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -217,7 +217,7 @@ static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* static int32_t compareStrInList(const void* pLeft, const void* pRight) { const SArray* arr = (const SArray*)pRight; - return taosArraySearchString(arr, &pLeft) == NULL ? 0 : 1; + return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1; } static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { From 8f7bd35a49a7c28e9331c56401c50fedac1d0702 Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 23 Apr 2020 16:21:19 +0800 Subject: [PATCH 33/63] TD-97: tbname criteria with other criteria is done --- src/query/src/qast.c | 10 +++++++--- src/tsdb/src/tsdbRead.c | 24 +++++++++++++++++------- src/util/src/tarray.c | 5 +++-- src/util/src/tcompare.c | 4 ++-- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 7b97e6f9c5..903d49a0d9 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -783,13 +783,17 @@ static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SBinaryFilte taosArrayCopy(pResult, array); } -static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, - SBinaryFilterSupp *param) { + +static void tSQLBinaryTraverseOnSkipList( + tExprNode *pExpr, + SArray *pResult, + SSkipList *pSkipList, + SBinaryFilterSupp *param +) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); while (tSkipListIterNext(iter)) { SSkipListNode *pNode = tSkipListIterGet(iter); - if (filterItem(pExpr, pNode, param)) { taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 58fd4eb653..4b565567ab 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1357,16 +1357,26 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - char* val = dataRowTuple(pTable->tagVal); // todo not only the first column + char* val = NULL; int8_t type = pInfo->sch.type; + + if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + val = pTable->name; + type = TSDB_DATA_TYPE_BINARY; + } else { + val = dataRowTuple(pTable->tagVal); // todo not only the first column + } int32_t ret = 0; - if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { - ret = pInfo->compare(val, pInfo->q.pz); + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (pInfo->optr == TSDB_RELATION_IN) { + ret = pInfo->compare(val, pInfo->q.arr); + } else { + ret = pInfo->compare(val, pInfo->q.pz); + } } else { tVariant t = {0}; tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type); - ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); } @@ -1393,7 +1403,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { return ret == 0; } case TSDB_RELATION_IN: { - + return ret == 1; } default: @@ -1474,8 +1484,8 @@ int32_t tsdbQueryByTagsCond( expr = calloc(1, sizeof(tExprNode)); expr->nodeType = TSQL_NODE_EXPR; expr->_node.optr = tagNameRelType; - expr->_node.pLeft = tbnameExpr; - expr->_node.pRight = tagExpr; + expr->_node.pLeft = tagExpr; + expr->_node.pRight = tbnameExpr; } } diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index ef57c2432f..51684e767c 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -77,10 +77,11 @@ void* taosArrayPush(SArray* pArray, void* pData) { } void* taosArrayPop(SArray* pArray) { - if (pArray == NULL || pArray->size == 0) { + assert( pArray != NULL ); + + if (pArray->size == 0) { return NULL; } - pArray->size -= 1; return TARRAY_GET_ELEM(pArray, pArray->size); } diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 98e0c9c9a2..4d78a8cf0a 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -215,7 +215,7 @@ static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -static int32_t compareStrInList(const void* pLeft, const void* pRight) { +static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) { const SArray* arr = (const SArray*)pRight; return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1; } @@ -277,7 +277,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } else if (optr == TSDB_RELATION_IN) { assert(filterDataType == TSDB_DATA_TYPE_ARRAY); - comparFn = compareStrInList; + comparFn = compareFindStrInArray; } else { /* normal relational comparFn */ assert(filterDataType == TSDB_DATA_TYPE_BINARY); From 7bb402ea78a061c1a14992469bf6cc0cd112b036 Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 23 Apr 2020 18:22:11 +0800 Subject: [PATCH 34/63] fix compile error --- src/query/src/queryExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index cb3bf86ebe..b9b304cfc1 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5273,7 +5273,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, - char **tagCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { + char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); From 8317e1a6ad979c88e809dd455533c17889f93b2d Mon Sep 17 00:00:00 2001 From: localvar Date: Thu, 23 Apr 2020 19:08:35 +0800 Subject: [PATCH 35/63] TD-97: extract table name from id --- src/query/src/qast.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 903d49a0d9..83a50c469d 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -1199,7 +1199,9 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN; for (const char *e = cond; *e != 0; e++) { - if (*e == ',') { + if (*e == TS_PATH_DELIMITER[0]) { + cond = e + 1; + } else if (*e == ',') { size_t len = e - cond + 1; char* p = malloc( len ); memcpy(p, cond, len); @@ -1210,10 +1212,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { } if (*cond != 0) { - size_t len = strlen(cond) + 1; - char* p = malloc( len ); - memcpy(p, cond, len); - p[len - 1] = 0; + char* p = strdup( cond ); taosArrayPush(pVal->arr, &p); } From 5ee943590c27a30650e97784bd4b04e0b5a3de95 Mon Sep 17 00:00:00 2001 From: localvar Date: Fri, 24 Apr 2020 10:00:14 +0800 Subject: [PATCH 36/63] fix memory leaks --- src/query/src/qast.c | 3 +++ src/tsdb/src/tsdbRead.c | 1 + 2 files changed, 4 insertions(+) diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 83a50c469d..70e4dbd803 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -798,6 +798,7 @@ static void tSQLBinaryTraverseOnSkipList( taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); } } + tSkipListDestroyIter(iter); } @@ -825,6 +826,8 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, taosArrayPush(result, (void*)&table); } } + + tSkipListDestroyIter(iter); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 4b565567ab..0449f42eb5 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1436,6 +1436,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) convertQueryResult(pRes, pTableList); taosArrayDestroy(pTableList); + free(schema); return TSDB_CODE_SUCCESS; } From a9e6abc09f6472c560c2fc5c78a4ec118798dca9 Mon Sep 17 00:00:00 2001 From: localvar Date: Fri, 24 Apr 2020 11:02:22 +0800 Subject: [PATCH 37/63] TD-97: add test case --- tests/pytest/select/__init__.py | 0 tests/pytest/select/tbname.py | 72 +++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 tests/pytest/select/__init__.py create mode 100644 tests/pytest/select/tbname.py diff --git a/tests/pytest/select/__init__.py b/tests/pytest/select/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/select/tbname.py b/tests/pytest/select/tbname.py new file mode 100644 index 0000000000..f088f9e63a --- /dev/null +++ b/tests/pytest/select/tbname.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.execute('create table cars (ts timestamp, speed int) tags(id int)') + tdSql.execute("create table carzero using cars tags(0)") + tdSql.execute("create table carone using cars tags(1)") + tdSql.execute("create table cartwo using cars tags(2)") + + tdSql.execute("insert into carzero values(now, 100) carone values(now, 110)") + + tdSql.query("select * from cars where tbname in ('carzero', 'carone')") + tdSql.checkRows(2) + + tdSql.query("select * from cars where tbname in ('carzero', 'cartwo')") + tdSql.checkRows(1) + + tdSql.query("select * from cars where id=1 or tbname in ('carzero0', 'cartwo')") + tdSql.checkRows(2) + + tdSql.query("select * from cars where id=1 and tbname in ('carzero0', 'cartwo')") + tdSql.checkRows(0) + + tdSql.query("select * from cars where id=0 and tbname in ('carzero0', 'cartwo')") + tdSql.checkRows(1) + + tdSql.query("select * from cars where tbname like 'car%') + tdSql.checkRows(2) + + tdSql.query("select * from cars where tbname like '%%o') + tdSql.checkRows(1) + + tdSql.query("select * from cars where id=1 and tbname like 'car%') + tdSql.checkRows(1) + + tdSql.query("select * from cars where id = 1 and tbname like '%%o') + tdSql.checkRows(0) + + tdSql.query("select * from cars where id = 1 or tbname like '%%o') + tdSql.checkRows(2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 7ddbf619800c5515cc126b363c40a083f252a5b1 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 24 Apr 2020 18:47:21 +0800 Subject: [PATCH 38/63] [td-187] support group by on normal columns. --- src/client/src/tscSQLParser.c | 6 +++++- src/query/src/queryExecutor.c | 38 ++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index df47afd0dc..588e45c0b1 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2477,6 +2477,10 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return TSDB_CODE_SUCCESS; } + if (pQueryInfo->colList == NULL) { + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + } + pQueryInfo->groupbyExpr.numOfGroupCols = pList->nExpr; if (pList->nExpr > TSDB_MAX_TAGS) { return invalidSqlErrMsg(pQueryInfo->msg, msg1); @@ -4915,7 +4919,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { list.num = 1; list.ids[0] = colIndex; - insertResultField(pQueryInfo, size - 1, &list, pSchema->bytes, pSchema->type, pSchema->name, pExpr); + insertResultField(pQueryInfo, size, &list, pSchema->bytes, pSchema->type, pSchema->name, pExpr); SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, size - 1); pInfo->visible = false; } diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index e2f3b2f655..1cb7918ed5 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -990,9 +990,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat return TSDB_CODE_SUCCESS; } -static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int16_t *type, int16_t *bytes) { - char *groupbyColumnData = NULL; - +static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, SArray* pDataBlock) { SSqlGroupbyExpr *pGroupbyExpr = pQuery->pGroupbyExpr; for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { @@ -1015,12 +1013,22 @@ static UNUSED_FUNC char *getGroupbyColumnData(SQuery *pQuery, SData **data, int1 *type = pQuery->colList[colIndex].type; *bytes = pQuery->colList[colIndex].bytes; - - // groupbyColumnData = doGetDataBlocks(pQuery, data, pQuery->colList[colIndex].inf); - break; + /* + * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare + * stage, the remain meter may not have the required column in cache actually. So, the validation of required + * column in cache with the corresponding meter schema is reinforced. + */ + int32_t numOfCols = taosArrayGetSize(pDataBlock); + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData *p = taosArrayGet(pDataBlock, i); + if (pColIndex->colId == p->info.colId) { + return p->pData; + } + } } - - return groupbyColumnData; + + return NULL; } static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { @@ -1091,8 +1099,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS char *groupbyColumnData = NULL; if (groupbyStateValue) { - assert(0); - // groupbyColumnData = getGroupbyColumnData(pQuery, data, &type, &bytes); + groupbyColumnData = getGroupbyColumnData(pQuery, &type, &bytes, pDataBlock); } for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { @@ -6088,9 +6095,16 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) STableId *id = taosArrayGet(pTableIdList, 0); id->uid = -1; // todo fix me - + + // group by normal column, do not pass the group by condition to tsdb to group table into different group + int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; + if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { + numOfGroupByCols = 0; + } + + // todo handle the error /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, &groupInfo, pGroupColIndex, - pQueryMsg->numOfGroupCols); + numOfGroupByCols); if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query code = TSDB_CODE_SUCCESS; goto _query_over; From f410f009608f2850967c8b354a7ed6d232ada238 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 24 Apr 2020 19:02:24 +0800 Subject: [PATCH 39/63] [TD-19] error code logic changed in http --- src/client/src/TSDBJNIConnector.c | 6 +- src/client/src/tscSystem.c | 10 +- src/inc/taoserror.h | 110 +++++++++--------- src/mnode/src/mgmtShell.c | 2 +- src/mnode/src/mgmtTable.c | 4 +- src/plugins/http/src/httpResp.c | 7 +- src/plugins/http/src/httpSql.c | 52 ++++----- src/plugins/http/src/tgJson.c | 14 +-- tests/script/general/http/restful.sim | 4 +- tests/script/general/http/restful_full.sim | 10 +- tests/script/general/http/testSuite.sim | 3 +- tests/script/sh/deploy.sh | 4 +- .../{general => unique}/http/opentsdb.sim | 0 tests/script/unique/http/testSuite.sim | 3 +- 14 files changed, 110 insertions(+), 119 deletions(-) rename tests/script/{general => unique}/http/opentsdb.sim (100%) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 29dec66ee8..05b7a2be1e 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -281,7 +281,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(J int code = taos_query(tscon, dst); if (code != 0) { - jniError("jobj:%p, conn:%p, code:%d, msg:%s", jobj, tscon, code, taos_errstr(tscon)); + jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, tscon, tstrerror(code), taos_errstr(tscon)); free(dst); return JNI_TDENGINE_ERROR; } else { @@ -290,9 +290,9 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(J if (pSql->cmd.command == TSDB_SQL_INSERT) { affectRows = taos_affected_rows(tscon); - jniTrace("jobj:%p, conn:%p, code:%d, affect rows:%d", jobj, tscon, code, affectRows); + jniTrace("jobj:%p, conn:%p, code:%s, affect rows:%d", jobj, tscon, tstrerror(code), affectRows); } else { - jniTrace("jobj:%p, conn:%p, code:%d", jobj, tscon, code); + jniTrace("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); } free(dst); diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 484c9e344e..fac771234a 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -42,7 +42,6 @@ void * tscTmr; void * tscQhandle; void * tscCheckDiskUsageTmr; int tsInsertHeadSize; -char tsLastUser[TSDB_USER_LEN + 1]; int tscNumOfThreads; @@ -81,13 +80,6 @@ int32_t tscInitRpc(const char *user, const char *secret) { } } - // not stop service, switch users - if (strcmp(tsLastUser, user) != 0 && pTscMgmtConn != NULL) { - tscTrace("switch user from %s to %s", user, tsLastUser); - rpcClose(pTscMgmtConn); - pTscMgmtConn = NULL; - } - if (pTscMgmtConn == NULL) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsLocalIp; @@ -101,8 +93,8 @@ int32_t tscInitRpc(const char *user, const char *secret) { rpcInit.idleTime = 2000; rpcInit.user = (char*)user; rpcInit.ckey = "key"; + rpcInit.spi = 1; rpcInit.secret = secretEncrypt; - strcpy(tsLastUser, user); pTscMgmtConn = rpcOpen(&rpcInit); if (pTscMgmtConn == NULL) { diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 7b060b1c65..5084199ac8 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -99,71 +99,71 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE, 0, 205, "not active tabl TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH, 0, 206, "table id mismatch") // dnode & mnode -TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 300, "no enough dnodes") -TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST, 0, 301, "dnode already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST, 0, 302, "dnode not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER, 0, 303, "no master") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER, 0, 304, "no remove master") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID, 0, 305, "invalid query id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID, 0, 306, "invalid stream id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION, 0, 307, "invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR, 0, 308, "sdb error") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 250, "no enough dnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_ALREADY_EXIST, 0, 251, "dnode already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_NOT_EXIST, 0, 252, "dnode not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_MASTER, 0, 253, "no master") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_REMOVE_MASTER, 0, 254, "no remove master") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_ID, 0, 255, "invalid query id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_STREAM_ID, 0, 256, "invalid stream id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CONNECTION, 0, 257, "invalid connection") +TAOS_DEFINE_ERROR(TSDB_CODE_SDB_ERROR, 0, 258, "sdb error") // acct -TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST, 0, 400, "accounts already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT, 0, 401, "invalid account") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER, 0, 402, "invalid account parameter") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS, 0, 403, "too many accounts") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS, 0, 404, "too many users") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES, 0, 405, "too many tables") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES, 0, 406, "too many databases") -TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES, 0, 407, "not enough time series") +TAOS_DEFINE_ERROR(TSDB_CODE_ACCT_ALREADY_EXIST, 0, 300, "accounts already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT, 0, 301, "invalid account") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_ACCT_PARAMETER, 0, 302, "invalid account parameter") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_ACCTS, 0, 303, "too many accounts") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_USERS, 0, 304, "too many users") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TABLES, 0, 305, "too many tables") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_DATABASES, 0, 306, "too many databases") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TIME_SERIES, 0, 307, "not enough time series") // grant -TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE, 0, 400, "auth failure") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS, 0, 401, "no rights") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS, 0, 402, "no write access") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS, 0, 403, "no read access") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 404, "grant expired") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 405, "grant dnode limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 406, "grant account limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 407, "grant timeseries limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 408, "grant db limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 409, "grant user limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 410, "grant conn limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 411, "grant stream limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 412, "grant speed limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 413, "grant storage limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 414, "grant query time limited") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 415, "grant cpu limited") +TAOS_DEFINE_ERROR(TSDB_CODE_AUTH_FAILURE, 0, 350, "auth failure") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_RIGHTS, 0, 351, "no rights") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_WRITE_ACCESS, 0, 352, "no write access") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_READ_ACCESS, 0, 353, "no read access") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 354, "grant expired") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 355, "grant dnode limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 356, "grant account limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 357, "grant timeseries limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 358, "grant db limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 359, "grant user limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 360, "grant conn limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 361, "grant stream limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 362, "grant speed limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 363, "grant storage limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 364, "grant query time limited") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 365, "grant cpu limited") // server -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID, 0, 500, "invalid vgroup id") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID, 0, 501, "invalid vnode id") -TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE, 0, 502, "not active vnode") -TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED, 0, 503, "vg init failed") -TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE, 0, 504, "server no diskspace") -TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY, 0, 505, "server out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 506, "no disk permissions") -TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, 0, 507, "file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 508, "memory corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VGROUP_ID, 0, 400, "invalid vgroup id") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VNODE_ID, 0, 401, "invalid vnode id") +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_VNODE, 0, 402, "not active vnode") +TAOS_DEFINE_ERROR(TSDB_CODE_VG_INIT_FAILED, 0, 403, "vg init failed") +TAOS_DEFINE_ERROR(TSDB_CODE_SERV_NO_DISKSPACE, 0, 404, "server no diskspace") +TAOS_DEFINE_ERROR(TSDB_CODE_SERV_OUT_OF_MEMORY, 0, 405, "server out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_DISK_PERMISSIONS, 0, 406, "no disk permissions") +TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, 0, 407, "file corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, 0, 408, "memory corrupted") // client -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION, 0, 601, "invalid client version") -TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY, 0, 602, "client out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE, 0, 603, "client no disk space") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP, 0, 604, "invalid timestamp") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL, 0, 605, "invalid sql") -TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED, 0, 606, "query cache erased") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG, 0, 607, "invalid query message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY, 0, 608, "sorted res too many") // too many result for ordered super table projection query -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 609, "invalid handle") -TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 610, "query cancelled") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 611, "invalid ie") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 612, "invalid value") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CLIENT_VERSION, 0, 451, "invalid client version") +TAOS_DEFINE_ERROR(TSDB_CODE_CLI_OUT_OF_MEMORY, 0, 452, "client out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_CLI_NO_DISKSPACE, 0, 453, "client no disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIME_STAMP, 0, 454, "invalid timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SQL, 0, 455, "invalid sql") +TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CACHE_ERASED, 0, 456, "query cache erased") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QUERY_MSG, 0, 457, "invalid query message") // failed to validate the sql expression msg by vnode +TAOS_DEFINE_ERROR(TSDB_CODE_SORTED_RES_TOO_MANY, 0, 458, "sorted res too many") // too many result for ordered super table projection query +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 459, "invalid handle") +TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 460, "query cancelled") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 461, "invalid ie") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 462, "invalid value") // others -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 700, "invalid file format") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 500, "invalid file format") #ifdef TAOS_ERROR_C diff --git a/src/mnode/src/mgmtShell.c b/src/mnode/src/mgmtShell.c index 927bbe4ed1..f2545f43db 100644 --- a/src/mnode/src/mgmtShell.c +++ b/src/mnode/src/mgmtShell.c @@ -364,7 +364,7 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *pMsg) { } static int mgmtShellRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - *spi = 0; + *spi = 1; *encrypt = 0; *ckey = 0; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index cad559b110..45db300e5f 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -1218,7 +1218,7 @@ static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) { } static void mgmtProcessDropSuperTableRsp(SRpcMsg *rpcMsg) { - mTrace("drop stable rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); + mTrace("drop stable rsp received, handle:%p code:%s", rpcMsg->handle, tstrerror(rpcMsg->code)); } static void *mgmtBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { @@ -1876,7 +1876,7 @@ static void mgmtProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { // not implemented yet static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg) { - mTrace("alter table rsp received, handle:%p code:%d", rpcMsg->handle, rpcMsg->code); + mTrace("alter table rsp received, handle:%p code:%s", rpcMsg->handle, tstrerror(rpcMsg->code)); } static void mgmtProcessMultiTableMetaMsg(SQueuedMsg *pMsg) { diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index b064b58116..0e2e7b3f85 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -44,9 +44,8 @@ const char *httpRespTemplate[] = { "%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sAccess-Control-Allow-Methods:POST, GET, OPTIONS, DELETE, PUT\r\nAccess-Control-Allow-Headers:Accept, Content-Type\r\nContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n" }; -void httpSendErrorRespImp(HttpContext *pContext, int httpCode, char *httpCodeStr, int errNo, char *desc) { - httpError("context:%p, fd:%d, ip:%s, code:%d, error:%d:%s", pContext, pContext->fd, pContext->ipstr, httpCode, errNo, - desc); +static void httpSendErrorRespImp(HttpContext *pContext, int httpCode, char *httpCodeStr, int errNo, char *desc) { + httpError("context:%p, fd:%d, ip:%s, code:%d, error:%s", pContext, pContext->fd, pContext->ipstr, httpCode, desc); char head[512] = {0}; char body[512] = {0}; @@ -184,7 +183,7 @@ void httpSendErrorResp(HttpContext *pContext, int errNo) { httpSendErrorRespWith void httpSendTaosdErrorResp(HttpContext *pContext, int errCode) { int httpCode = 400; - httpSendErrorRespImp(pContext, httpCode, "Bad Request", errCode, (char*)tstrerror(errCode)); + httpSendErrorRespImp(pContext, httpCode, "Bad Request", 1000, (char*)tstrerror(errCode)); } void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 22ecdbc7b2..0d0bc3e0fd 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -56,8 +56,8 @@ void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numO pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, numOfRows, sql); if (numOfRows < 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, retrieve failed code:%d, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, -numOfRows, sql); + httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(numOfRows), sql); } else { taos_free_result(result); } @@ -80,21 +80,21 @@ void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) { HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos; char * sql = httpGetCmdsString(pContext, singleCmd->sql); - if (-code == TSDB_CODE_ACTION_IN_PROGRESS) { - httpWarn("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, code:%d:inprogress, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, -code, sql); + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { + httpWarn("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, code:%s:inprogress, sql:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(code), sql); return; } if (code < 0) { - if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code >= 0 ? 0 : -code)) { - singleCmd->code = -code; - httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%d, last sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, -code, sql); + if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code >= 0 ? 0 : code)) { + singleCmd->code = code; + httpTrace("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%s, last sql:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, tstrerror(code), sql); } else { - singleCmd->code = -code; - httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, error code:%d, sql:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, -code, sql); + singleCmd->code = code; + httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, error code:%s, sql:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(code), sql); if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN) { if (encode->startJsonFp) (encode->startJsonFp)(pContext, singleCmd, result); @@ -210,8 +210,8 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num pContext->user, numOfRows); if (numOfRows < 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, retrieve failed, code:%d", pContext, pContext->fd, pContext->ipstr, - pContext->user, -numOfRows); + httpError("context:%p, fd:%d, ip:%s, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->ipstr, + pContext->user, tstrerror(numOfRows)); } else { taos_free_result(result); } @@ -230,22 +230,22 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) { HttpEncodeMethod *encode = pContext->encodeMethod; - if (-code == TSDB_CODE_ACTION_IN_PROGRESS) { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%d:inprogress, sqlObj:%p", - pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, -code, (SSqlObj *)result); + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { + httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s:inprogress, sqlObj:%p", + pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), (SSqlObj *)result); return; } if (code < 0) { SSqlObj *pObj = (SSqlObj *)result; - if (-code == TSDB_CODE_INVALID_SQL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%d:invalidsql, sqlObj:%p, error:%s", - pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, -code, pObj, pObj->cmd.payload); + if (code == TSDB_CODE_INVALID_SQL) { + httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s:invalidsql, sqlObj:%p, error:%s", + pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload); httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload); } else { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%d, sqlObj:%p", - pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, -code, pObj); - httpSendTaosdErrorResp(pContext, -code); + httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s, sqlObj:%p", + pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj); + httpSendTaosdErrorResp(pContext, code); } return; } @@ -350,9 +350,9 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int code) { if (pContext == NULL || pContext->signature != pContext) return; if (code < 0) { - httpError("context:%p, fd:%d, ip:%s, user:%s, login error, code:%d", pContext, pContext->fd, pContext->ipstr, - pContext->user, -code); - httpSendTaosdErrorResp(pContext, -code); + httpError("context:%p, fd:%d, ip:%s, user:%s, login error, code:%s", pContext, pContext->fd, pContext->ipstr, + pContext->user, tstrerror(code)); + httpSendTaosdErrorResp(pContext, code); return; } diff --git a/src/plugins/http/src/tgJson.c b/src/plugins/http/src/tgJson.c index 1513d61d82..cb9c42e792 100644 --- a/src/plugins/http/src/tgJson.c +++ b/src/plugins/http/src/tgJson.c @@ -98,8 +98,8 @@ void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { HttpSqlCmds *multiCmds = pContext->multiCmds; - httpTrace("context:%p, fd:%d, ip:%s, check telegraf command, code:%d, state:%d, type:%d, rettype:%d, tags:%d", - pContext, pContext->fd, pContext->ipstr, code, cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); + httpTrace("context:%p, fd:%d, ip:%s, check telegraf command, code:%s, state:%d, type:%d, rettype:%d, tags:%d", + pContext, pContext->fd, pContext->ipstr, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) { if (cmd->cmdState == HTTP_CMD_STATE_NOT_RUN_YET) { @@ -125,11 +125,11 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { } } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_DB) { cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - httpTrace("context:%p, fd:%d, ip:%s, code:%d, create database failed", pContext, pContext->fd, pContext->ipstr, - code); + httpTrace("context:%p, fd:%d, ip:%s, code:%s, create database failed", pContext, pContext->fd, pContext->ipstr, + tstrerror(code)); } else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_STBALE) { cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED; - httpTrace("context:%p, fd:%d, ip:%s, code:%d, create stable failed", pContext, pContext->fd, pContext->ipstr, code); + httpTrace("context:%p, fd:%d, ip:%s, code:%s, create stable failed", pContext, pContext->fd, pContext->ipstr, tstrerror(code)); } else { } @@ -138,8 +138,8 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) { HttpSqlCmds *multiCmds = pContext->multiCmds; - httpTrace("context:%p, fd:%d, ip:%s, get telegraf next command, pos:%d, code:%d, state:%d, type:%d, rettype:%d, tags:%d", - pContext, pContext->fd, pContext->ipstr, multiCmds->pos, code, cmd->cmdState, cmd->cmdType, + httpTrace("context:%p, fd:%d, ip:%s, get telegraf next command, pos:%d, code:%s, state:%d, type:%d, rettype:%d, tags:%d", + pContext, pContext->fd, pContext->ipstr, multiCmds->pos, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum); if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) { diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index 4c0adae41d..50f0223804 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -77,8 +77,8 @@ if $system_content != @{"status":"succ","code":0,"desc":"jIlItaLFFIPa8qdtNZmtONr return -1 endi -system_content curl 127.0.0.1:6020/rest/login/u2/abcd_1234 +system_content curl 127.0.0.1:6020/rest/login/u2/aabcd_1234 print curl 127.0.0.1:6020/rest/login/u2/abcd_1234 -----> $system_content -if $system_content != @{"status":"succ","code":0,"desc":"AZH3lNvWZiLa8qdtNZmtONryp201ma04QExCHYkbFFQB/46k1gwnPNryp201ma04"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then return -1 endi diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 2be125021d..6682aac352 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -1,13 +1,13 @@ system sh/stop_dnodes.sh -sleep 5000 +#sleep 5000 system sh/ip.sh -i 1 -s up system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1 system sh/cfg.sh -n dnode1 -c clog -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +#sleep 3000 sql connect print ============================ dnode1 start @@ -41,19 +41,19 @@ endi system_content curl 192.168.0.1:6020/rest/login/root/123 print 5-> $system_content -if $system_content != @{"status":"error","code":10,"desc":"authentication failure"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then return -1 endi system_content curl 192.168.0.1:6020/rest/login/root/123/1/1/3 print 6-> $system_content -if $system_content != @{"status":"error","code":10,"desc":"authentication failure"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 192.168.0.1:6020/rest/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":10,"desc":"authentication failure"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"auth failure"}@ then return -1 endi diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index ca93dedd3b..092eebdbdf 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -1,7 +1,6 @@ -run general/http/grafana.sim run general/http/restful.sim run general/http/restful_full.sim run general/http/telegraf.sim run general/http/prepare.sim run general/http/grafana_bug.sim -#run general/http/opentsdb.sim +run general/http/grafana.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 41ba3c425a..907ca6848b 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -93,10 +93,10 @@ echo "privateIp $NODE_IP" >> $TAOS_CFG echo "dDebugFlag 199" >> $TAOS_CFG echo "mDebugFlag 199" >> $TAOS_CFG echo "sdbDebugFlag 199" >> $TAOS_CFG -echo "rpcDebugFlag 135" >> $TAOS_CFG +echo "rpcDebugFlag 131" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG -echo "httpDebugFlag 131" >> $TAOS_CFG +echo "httpDebugFlag 135" >> $TAOS_CFG echo "monitorDebugFlag 131" >> $TAOS_CFG echo "udebugFlag 131" >> $TAOS_CFG echo "jnidebugFlag 131" >> $TAOS_CFG diff --git a/tests/script/general/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim similarity index 100% rename from tests/script/general/http/opentsdb.sim rename to tests/script/unique/http/opentsdb.sim diff --git a/tests/script/unique/http/testSuite.sim b/tests/script/unique/http/testSuite.sim index b3ee1792f5..d4a6b1f5f5 100644 --- a/tests/script/unique/http/testSuite.sim +++ b/tests/script/unique/http/testSuite.sim @@ -1 +1,2 @@ -run unique/http/admin.sim \ No newline at end of file +run unique/http/admin.sim +#run general/http/opentsdb.sim \ No newline at end of file From 59c6434a0d908ad529b7e44fe68b1bde4b75b5f8 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Fri, 24 Apr 2020 22:32:23 +0800 Subject: [PATCH 40/63] [td-187] 1. support group by query, 2. fix memory leaks in client side. 3. refactor mgmtConn --- src/client/inc/tscUtil.h | 4 ++-- src/client/inc/tsclient.h | 5 ++--- src/client/src/tscAsync.c | 2 +- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscServer.c | 5 +++-- src/client/src/tscSql.c | 4 +++- src/client/src/tscStream.c | 6 +++--- src/client/src/tscSystem.c | 22 +++++++--------------- src/client/src/tscUtil.c | 28 ++++++++++++++++++++-------- tests/examples/c/demo.c | 4 ++-- 10 files changed, 44 insertions(+), 38 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index c16945bbcf..d46c32d73d 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -196,14 +196,14 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); -void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); -void tscFreeSubqueryInfo(SSqlCmd* pCmd); +void tscFreeQueryInfo(SSqlCmd* pCmd); void tscClearSubqueryInfo(SSqlCmd* pCmd); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 9cc6bcb364..d6fb9279c7 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -298,6 +298,7 @@ typedef struct STscObj { char sversion[TSDB_VERSION_LEN]; char writeAuth : 1; char superAuth : 1; + void* pMgmtConn; struct SSqlObj * pSql; struct SSqlObj * pHb; struct SSqlObj * sqlList; @@ -359,7 +360,7 @@ typedef struct SSqlStream { struct SSqlStream *prev, *next; } SSqlStream; -int32_t tscInitRpc(const char *user, const char *secret); +int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn); void tscInitMsgsFp(); int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion); @@ -427,9 +428,7 @@ int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column); extern void * pVnodeConn; -extern void * pTscMgmtConn; extern void * tscCacheHandle; -extern int slaveIndex; extern void * tscTmr; extern void * tscQhandle; extern int tscKeepConn[]; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 99d20de48b..b954db0734 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -46,7 +46,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; - pSql->maxRetry = 1; + pSql->maxRetry = TSDB_REPLICA_MAX_NUM; pSql->fp = fp; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 588e45c0b1..59e2f147e1 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -733,7 +733,7 @@ int32_t tscSetTableId(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SS */ if (size > 0) { if (strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) { - tscClearMeterMetaInfo(pTableMetaInfo, false); + tscClearTableMetaInfo(pTableMetaInfo, false); } } else { assert(pTableMetaInfo->pTableMeta == NULL); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5ae8f2402b..c36102034b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -188,6 +188,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { } int tscSendMsgToServer(SSqlObj *pSql) { + STscObj* pObj = pSql->pTscObj; SSqlCmd* pCmd = &pSql->cmd; char *pMsg = rpcMallocCont(pCmd->payloadLen); @@ -221,7 +222,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = pSql, .code = 0 }; - rpcSendRequest(pTscMgmtConn, &pSql->ipList, &rpcMsg); + rpcSendRequest(pObj->pMgmtConn, &pSql->ipList, &rpcMsg); } return TSDB_CODE_SUCCESS; @@ -2597,7 +2598,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { // if (pSql->fp != NULL && pSql->pStream == NULL) { // pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); -// tscFreeSubqueryInfo(pCmd); +// tscFreeQueryInfo(pCmd); // } tscTrace("%p allocate new pSqlObj:%p to get stable vgroupInfo", pSql, pNew); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index f70beab166..4320e8f81b 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -66,7 +66,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con return NULL; } - if (tscInitRpc(user, pass) != 0) { + void* pMgmtConn = NULL; + if (tscInitRpc(user, pass, &pMgmtConn) != 0) { terrno = TSDB_CODE_NETWORK_UNAVAIL; return NULL; } @@ -118,6 +119,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con strtolower(pObj->db, tmp); } + pObj->pMgmtConn = pMgmtConn; pthread_mutex_init(&pObj->mutex, NULL); SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 9f7d4887d1..d690681729 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -147,7 +147,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - tscClearMeterMetaInfo(pTableMetaInfo, true); + tscClearTableMetaInfo(pTableMetaInfo, true); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); return; @@ -177,7 +177,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf if (pSql == NULL || numOfRows < 0) { int64_t retryDelayTime = tscGetRetryDelayTime(pStream->slidingTime, pStream->precision); tscError("%p stream:%p, retrieve data failed, code:%d, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retryDelayTime); - tscClearMeterMetaInfo(pTableMetaInfo, true); + tscClearTableMetaInfo(pTableMetaInfo, true); tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime); return; @@ -259,7 +259,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf pStream->numOfRes); // release the metric/meter meta information reference, so data in cache can be updated - tscClearMeterMetaInfo(pTableMetaInfo, false); + tscClearTableMetaInfo(pTableMetaInfo, false); tscSetNextLaunchTimer(pStream, pSql); } } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 6713f84f99..6966a58c45 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -33,9 +33,6 @@ // global, not configurable void * pVnodeConn; -void * pVMeterConn; -void * pTscMgmtConn; -void * pSlaveConn; void * tscCacheHandle; int slaveIndex; void * tscTmr; @@ -55,7 +52,7 @@ void tscCheckDiskUsage(void *para, void *unused) { taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); } -int32_t tscInitRpc(const char *user, const char *secret) { +int32_t tscInitRpc(const char *user, const char *secret, void** pMgmtConn) { SRpcInit rpcInit; char secretEncrypt[32] = {0}; taosEncryptPass((uint8_t *)secret, strlen(secret), secretEncrypt); @@ -83,13 +80,13 @@ int32_t tscInitRpc(const char *user, const char *secret) { } // not stop service, switch users - if (strcmp(tsLastUser, user) != 0 && pTscMgmtConn != NULL) { + if (strcmp(tsLastUser, user) != 0 && *pMgmtConn != NULL) { tscTrace("switch user from %s to %s", user, tsLastUser); - rpcClose(pTscMgmtConn); - pTscMgmtConn = NULL; + rpcClose(*pMgmtConn); + *pMgmtConn = NULL; } - if (pTscMgmtConn == NULL) { + if (*pMgmtConn == NULL) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localIp = tsLocalIp; rpcInit.localPort = 0; @@ -104,8 +101,8 @@ int32_t tscInitRpc(const char *user, const char *secret) { rpcInit.secret = secretEncrypt; strcpy(tsLastUser, user); - pTscMgmtConn = rpcOpen(&rpcInit); - if (pTscMgmtConn == NULL) { + *pMgmtConn = rpcOpen(&rpcInit); + if (*pMgmtConn == NULL) { tscError("failed to init connection to mgmt"); return -1; } @@ -219,11 +216,6 @@ void taos_cleanup() { pVnodeConn = NULL; } - if (pTscMgmtConn != NULL) { - rpcClose(pTscMgmtConn); - pTscMgmtConn = NULL; - } - taosTmrCleanUp(tscTmr); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index abdab2d9a1..e9ff558b92 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -337,7 +337,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd) { pCmd->pTableList= NULL; pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeSubqueryInfo(pCmd); + tscFreeQueryInfo(pCmd); } /* @@ -761,6 +761,8 @@ void tscCloseTscObj(STscObj* pObj) { tscFreeSqlObj(pSql); sem_destroy(&pSql->rspSem); + rpcClose(pObj->pMgmtConn); + pthread_mutex_destroy(&pObj->mutex); tscTrace("%p DB connection is closed", pObj); @@ -1459,7 +1461,7 @@ bool tscShouldFreeHeatBeat(SSqlObj* pHb) { void tscCleanSqlCmd(SSqlCmd* pCmd) { pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeSubqueryInfo(pCmd); + tscFreeQueryInfo(pCmd); uint32_t allocSize = pCmd->allocSize; char* allocPtr = pCmd->payload; @@ -1601,7 +1603,7 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { return TSDB_CODE_SUCCESS; } -static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { +static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { tscTagCondRelease(&pQueryInfo->tagCond); tscFieldInfoClear(&pQueryInfo->fieldsInfo); @@ -1611,6 +1613,11 @@ static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { tscColumnListDestroy(pQueryInfo->colList); memset(&pQueryInfo->colList, 0, sizeof(pQueryInfo->colList)); + if (pQueryInfo->groupbyExpr.columnInfo != NULL) { + taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo); + pQueryInfo->groupbyExpr.columnInfo = NULL; + } + pQueryInfo->tsBuf = tsBufDestory(pQueryInfo->tsBuf); tfree(pQueryInfo->defaultVal); @@ -1619,11 +1626,11 @@ static void doClearSubqueryInfo(SQueryInfo* pQueryInfo) { void tscClearSubqueryInfo(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->numOfClause; ++i) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); - doClearSubqueryInfo(pQueryInfo); + freeQueryInfoImpl(pQueryInfo); } } -void tscFreeSubqueryInfo(SSqlCmd* pCmd) { +void tscFreeQueryInfo(SSqlCmd* pCmd) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } @@ -1632,7 +1639,7 @@ void tscFreeSubqueryInfo(SSqlCmd* pCmd) { char* addr = (char*)pCmd - offsetof(SSqlObj, cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); - doClearSubqueryInfo(pQueryInfo); + freeQueryInfoImpl(pQueryInfo); tscClearAllTableMetaInfo(pQueryInfo, (const char*)addr, false); tfree(pQueryInfo); } @@ -1691,7 +1698,7 @@ void doRemoveTableMetaInfo(SQueryInfo* pQueryInfo, int32_t index, bool removeFro STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index); - tscClearMeterMetaInfo(pTableMetaInfo, removeFromCache); + tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); free(pTableMetaInfo); int32_t after = pQueryInfo->numOfTables - index - 1; @@ -1713,13 +1720,18 @@ void tscClearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool tfree(pQueryInfo->pTableMetaInfo); } -void tscClearMeterMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { if (pTableMetaInfo == NULL) { return; } taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); tfree(pTableMetaInfo->vgroupList); + + if (pTableMetaInfo->tagColList != NULL) { + taosArrayDestroy(pTableMetaInfo->tagColList); + pTableMetaInfo->tagColList = NULL; + } } void tscResetForNextRetrieve(SSqlRes* pRes) { diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 9847300024..e5a70740f1 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -75,7 +75,7 @@ int main(int argc, char *argv[]) { doQuery(taos, "create database if not exists test"); doQuery(taos, "use test"); - doQuery(taos, "insert into tm99 values('2020-01-01 1:1:1', 99);"); + doQuery(taos, "select count(*),k,sum(k) from m1 group by k"); // doQuery(taos, "create table if not exists tm0 (ts timestamp, k int);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:1', 1);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:2', 2);"); @@ -86,7 +86,7 @@ int main(int argc, char *argv[]) { // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:7', 7);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:8', 8);"); // doQuery(taos, "insert into tm0 values('2020-1-1 1:1:9', 9);"); - doQuery(taos, "select sum(k),count(*) from m1 group by a"); +// doQuery(taos, "select sum(k),count(*) from m1 group by a"); taos_close(taos); return 0; From f62e4b957a630df8447ce137eb084891b4d84a36 Mon Sep 17 00:00:00 2001 From: slguan Date: Fri, 24 Apr 2020 23:14:12 +0800 Subject: [PATCH 41/63] [TD-183] --- src/common/inc/tglobal.h | 128 ++++++++-------- src/common/src/tglobal.c | 80 +++------- src/inc/taosdef.h | 7 +- src/mnode/inc/mgmtDef.h | 48 +++--- src/mnode/src/mgmtDb.c | 298 +++++++++++++++---------------------- src/mnode/src/mgmtTable.c | 16 +- src/mnode/src/mgmtVgroup.c | 32 ++-- 7 files changed, 249 insertions(+), 360 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index a19331c706..f770c8d38f 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -48,11 +48,9 @@ extern int32_t tsEnableCoreFile; extern int32_t tsTotalMemoryMB; extern int32_t tsVersion; -extern int tscEmbedded; - +extern int32_t tscEmbedded; extern int64_t tsMsPerDay[2]; - extern char tsMasterIp[]; extern char tsSecondIp[]; extern uint16_t tsMnodeDnodePort; @@ -61,94 +59,88 @@ extern uint16_t tsDnodeShellPort; extern uint16_t tsDnodeMnodePort; extern uint16_t tsSyncPort; -extern int tsStatusInterval; -extern int tsShellActivityTimer; -extern int tsVnodePeerHBTimer; -extern int tsMgmtPeerHBTimer; -extern int tsMeterMetaKeepTimer; -extern int tsMetricMetaKeepTimer; +extern int32_t tsStatusInterval; +extern int32_t tsShellActivityTimer; +extern int32_t tsVnodePeerHBTimer; +extern int32_t tsMgmtPeerHBTimer; +extern int32_t tsMeterMetaKeepTimer; +extern int32_t tsMetricMetaKeepTimer; -extern float tsNumOfThreadsPerCore; -extern float tsRatioOfQueryThreads; -extern char tsPublicIp[]; -extern char tsPrivateIp[]; -extern short tsNumOfVnodesPerCore; -extern short tsNumOfTotalVnodes; -extern short tsCheckHeaderFile; +extern float tsNumOfThreadsPerCore; +extern float tsRatioOfQueryThreads; +extern char tsPublicIp[]; +extern char tsPrivateIp[]; +extern int16_t tsNumOfVnodesPerCore; +extern int16_t tsNumOfTotalVnodes; extern uint32_t tsPublicIpInt; -extern short tsAffectedRowsMod; -extern int tsSessionsPerVnode; -extern int tsAverageCacheBlocks; -extern int tsCacheBlockSize; +extern int32_t tsMaxCacheSize; +extern int32_t tsSessionsPerVnode; +extern int16_t tsDaysPerFile; +extern int32_t tsDaysToKeep; +extern int32_t tsRowsInFileBlock; +extern int16_t tsCommitTime; // seconds +extern int32_t tsTimePrecision; +extern int16_t tsCompression; +extern int16_t tsCommitLog; +extern int32_t tsReplications; -extern int tsRowsInFileBlock; -extern float tsFileBlockMinPercent; - -extern short tsNumOfBlocksPerMeter; -extern short tsCommitTime; // seconds -extern short tsCommitLog; -extern short tsAsyncLog; -extern short tsCompression; -extern short tsDaysPerFile; -extern int tsDaysToKeep; -extern int tsReplications; - -extern int tsNumOfMPeers; -extern int tsMaxShellConns; -extern int tsMaxTables; +extern int16_t tsAffectedRowsMod; +extern int32_t tsNumOfMPeers; +extern int32_t tsMaxShellConns; +extern int32_t tsMaxTables; extern char tsLocalIp[]; extern char tsDefaultDB[]; extern char tsDefaultUser[]; extern char tsDefaultPass[]; -extern int tsMaxMeterConnections; -extern int tsMaxVnodeConnections; -extern int tsMaxMgmtConnections; +extern int32_t tsMaxMeterConnections; +extern int32_t tsMaxVnodeConnections; +extern int32_t tsMaxMgmtConnections; -extern int tsBalanceMonitorInterval; -extern int tsBalanceStartInterval; -extern int tsBalancePolicy; -extern int tsOfflineThreshold; -extern int tsMgmtEqualVnodeNum; +extern int32_t tsBalanceMonitorInterval; +extern int32_t tsBalanceStartInterval; +extern int32_t tsOfflineThreshold; +extern int32_t tsMgmtEqualVnodeNum; -extern int tsEnableHttpModule; -extern int tsEnableMonitorModule; -extern int tsRestRowLimit; -extern int tsCompressMsgSize; -extern int tsMaxSQLStringLen; -extern int tsMaxNumOfOrderedResults; +extern int32_t tsEnableHttpModule; +extern int32_t tsEnableMonitorModule; + +extern int32_t tsRestRowLimit; +extern int32_t tsMaxSQLStringLen; +extern int32_t tsCompressMsgSize; +extern int32_t tsMaxNumOfOrderedResults; extern char tsSocketType[4]; -extern int tsTimePrecision; -extern int tsMinSlidingTime; -extern int tsMinIntervalTime; -extern int tsMaxStreamComputDelay; -extern int tsStreamCompStartDelay; -extern int tsStreamCompRetryDelay; +extern int32_t tsMinSlidingTime; +extern int32_t tsMinIntervalTime; +extern int32_t tsMaxStreamComputDelay; +extern int32_t tsStreamCompStartDelay; +extern int32_t tsStreamCompRetryDelay; extern float tsStreamComputDelayRatio; // the delayed computing ration of the whole time window extern int tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -extern char tsHttpIp[]; +extern char tsHttpIp[]; extern uint16_t tsHttpPort; -extern int tsHttpCacheSessions; -extern int tsHttpSessionExpire; -extern int tsHttpMaxThreads; -extern int tsHttpEnableCompress; -extern int tsHttpEnableRecordSql; -extern int tsTelegrafUseFieldNum; +extern int32_t tsHttpCacheSessions; +extern int32_t tsHttpSessionExpire; +extern int32_t tsHttpMaxThreads; +extern int32_t tsHttpEnableCompress; +extern int32_t tsHttpEnableRecordSql; +extern int32_t tsTelegrafUseFieldNum; -extern int tsTscEnableRecordSql; -extern int tsAnyIp; +extern int32_t tsTscEnableRecordSql; +extern int32_t tsAnyIp; -extern char tsMonitorDbName[]; -extern char tsInternalPass[]; -extern int tsMonitorInterval; +extern char tsMonitorDbName[]; +extern char tsInternalPass[]; +extern int32_t tsMonitorInterval; -extern int tsNumOfLogLines; +extern int32_t tsAsyncLog; +extern int32_t tsNumOfLogLines; extern int32_t ddebugFlag; extern int32_t mdebugFlag; extern int32_t cdebugFlag; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 4998d43518..5aa50677c9 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -74,16 +74,15 @@ int32_t tsVnodePeerHBTimer = 1; // second int32_t tsMgmtPeerHBTimer = 1; // second int32_t tsMeterMetaKeepTimer = 7200; // second int32_t tsMetricMetaKeepTimer = 600; // second -int tsRpcTimer = 300; -int tsRpcMaxTime = 600; // seconds; +int32_t tsRpcTimer = 300; +int32_t tsRpcMaxTime = 600; // seconds; -float tsNumOfThreadsPerCore = 1.0; -float tsRatioOfQueryThreads = 0.5; -char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0}; -char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0}; +float tsNumOfThreadsPerCore = 1.0; +float tsRatioOfQueryThreads = 0.5; +char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0}; +char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0}; int16_t tsNumOfVnodesPerCore = 8; int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM; -int16_t tsCheckHeaderFile = 0; #ifdef _TD_ARM_32_ int32_t tsSessionsPerVnode = 100; @@ -91,26 +90,22 @@ int32_t tsSessionsPerVnode = 100; int32_t tsSessionsPerVnode = 1000; #endif -int32_t tsCacheBlockSize = 16384; // 256 columns -int32_t tsAverageCacheBlocks = TSDB_DEFAULT_AVG_BLOCKS; +int32_t tsMaxCacheSize = 64; //64M +int16_t tsDaysPerFile = 10; +int32_t tsDaysToKeep = 3650; +int32_t tsRowsInFileBlock = 4096; +int16_t tsCommitTime = 3600; // seconds +int32_t tsTimePrecision = TSDB_TIME_PRECISION_MILLI; +int16_t tsCompression = TSDB_MAX_COMPRESSION_LEVEL; +int16_t tsCommitLog = 1; +int32_t tsReplications = TSDB_REPLICA_MIN_NUM; + /** * Change the meaning of affected rows: * 0: affected rows not include those duplicate records * 1: affected rows include those duplicate records */ int16_t tsAffectedRowsMod = 0; - -int32_t tsRowsInFileBlock = 4096; -float tsFileBlockMinPercent = 0.05; - -int16_t tsNumOfBlocksPerMeter = 100; -int16_t tsCommitTime = 3600; // seconds -int16_t tsCommitLog = 1; -int16_t tsCompression = TSDB_MAX_COMPRESSION_LEVEL; -int16_t tsDaysPerFile = 10; -int32_t tsDaysToKeep = 3650; -int32_t tsReplications = TSDB_REPLICA_MIN_NUM; - int32_t tsNumOfMPeers = 3; int32_t tsMaxShellConns = 2000; int32_t tsMaxTables = 100000; @@ -125,15 +120,16 @@ int32_t tsMaxVnodeConnections = 10000; int32_t tsBalanceMonitorInterval = 2; // seconds int32_t tsBalanceStartInterval = 300; // seconds -int32_t tsBalancePolicy = 0; // 1-use sys.montor int32_t tsOfflineThreshold = 864000; // seconds 10days int32_t tsMgmtEqualVnodeNum = 4; int32_t tsEnableHttpModule = 1; int32_t tsEnableMonitorModule = 0; + int32_t tsRestRowLimit = 10240; int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN; +int32_t tsNumOfLogLines = 10000000; int32_t mdebugFlag = 135; int32_t sdbDebugFlag = 135; int32_t ddebugFlag = 131; @@ -146,7 +142,6 @@ int32_t qdebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t uDebugFlag = 131; int32_t debugFlag = 131; -int tsNumOfLogLines = 10000000; // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp @@ -165,9 +160,6 @@ int32_t tsCompressMsgSize = -1; // use UDP by default[option: udp, tcp] char tsSocketType[4] = "udp"; -// time precision, millisecond by default -int32_t tsTimePrecision = TSDB_TIME_PRECISION_MILLI; - // 10 ms for sliding time, the value will changed in case of time precision changed int32_t tsMinSlidingTime = 10; @@ -747,11 +739,11 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); cfg.option = "cache"; - cfg.ptr = &tsCacheBlockSize; + cfg.ptr = &tsMaxCacheSize; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 100; - cfg.maxValue = 1048576; + cfg.minValue = 1; + cfg.maxValue = 100000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); @@ -766,36 +758,6 @@ static void doInitGlobalConfig() { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "fileBlockMinPercent"; - cfg.ptr = &tsFileBlockMinPercent; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1.0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "ablocks"; - cfg.ptr = &tsAverageCacheBlocks; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_AVG_BLOCKS; - cfg.maxValue = TSDB_MAX_AVG_BLOCKS; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "tblocks"; - cfg.ptr = &tsNumOfBlocksPerMeter; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 32; - cfg.maxValue = 4096; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - // login configs cfg.option = "defaultDB"; cfg.ptr = tsDefaultDB; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 1f64a0d5c1..216de90598 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -218,9 +218,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS+1) #define TSDB_REPLICA_MIN_NUM 1 -/* - * this is defined in CMakeList.txt - */ #define TSDB_REPLICA_MAX_NUM 3 #define TSDB_TBNAME_COLUMN_INDEX (-1) @@ -242,8 +239,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MIN_ROWS_IN_FILEBLOCK 200 #define TSDB_MAX_ROWS_IN_FILEBLOCK 500000 -#define TSDB_MIN_CACHE_BLOCK_SIZE 100 -#define TSDB_MAX_CACHE_BLOCK_SIZE 104857600 +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 1000000 #define TSDB_MIN_CACHE_BLOCKS 100 #define TSDB_MAX_CACHE_BLOCKS 409600 diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index 5599ad2e4f..6cbfc72e62 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -29,32 +29,6 @@ struct SAcctObj; struct SUserObj; struct SMnodeObj; -typedef struct { - char acct[TSDB_USER_LEN]; - char db[TSDB_DB_NAME_LEN]; - uint32_t vgId; - int32_t maxSessions; - int32_t cacheBlockSize; - union { - int32_t totalBlocks; - float fraction; - } cacheNumOfBlocks; - int32_t daysPerFile; - int32_t daysToKeep1; - int32_t daysToKeep2; - int32_t daysToKeep; - int32_t commitTime; - int32_t rowsInFileBlock; - int16_t blocksPerTable; - int8_t compression; - int8_t commitLog; - int8_t replications; - int8_t repStrategy; - int8_t loadLatest; // load into mem or not - uint8_t precision; // time resolution - int8_t reserved[16]; -} SDbCfg; - typedef struct SDnodeObj { int32_t dnodeId; uint32_t privateIp; @@ -168,12 +142,30 @@ typedef struct SVgObj { SChildTableObj **tableList; } SVgObj; +typedef struct { + int64_t maxCacheSize; + int32_t maxTables; + int32_t daysPerFile; + int32_t daysToKeep; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t minRowsPerFileBlock; // minimum rows per file block + int32_t maxRowsPerFileBlock; // maximum rows per file block + int32_t commitTime; + int8_t precision; + int8_t compression; + int8_t commitLog; + int8_t replications; + int8_t reserved[16]; +} SDbCfg; + typedef struct SDbObj { char name[TSDB_DB_NAME_LEN + 1]; - int8_t status; + char acct[TSDB_USER_LEN + 1]; int64_t createdTime; SDbCfg cfg; - int8_t reserved[15]; + int8_t status; + int8_t reserved[14]; int8_t updateEnd[1]; int32_t refCount; int32_t numOfVgroups; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 1a24a9661b..b3424ffd63 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -54,7 +54,7 @@ static int32_t mgmtDbActionDestroy(SSdbOper *pOper) { static int32_t mgmtDbActionInsert(SSdbOper *pOper) { SDbObj *pDb = pOper->pObj; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); pDb->pHead = NULL; pDb->pTail = NULL; @@ -67,7 +67,7 @@ static int32_t mgmtDbActionInsert(SSdbOper *pOper) { mgmtDecAcctRef(pAcct); } else { - mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->cfg.acct); + mError("db:%s, acct:%s info not exist in sdb", pDb->name, pDb->acct); return TSDB_CODE_INVALID_ACCT; } @@ -76,7 +76,7 @@ static int32_t mgmtDbActionInsert(SSdbOper *pOper) { static int32_t mgmtDbActionDelete(SSdbOper *pOper) { SDbObj *pDb = pOper->pObj; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); mgmtDropDbFromAcct(pAcct, pDb); mgmtDropAllChildTables(pDb); @@ -176,132 +176,108 @@ SDbObj *mgmtGetDbByTableId(char *tableId) { return (SDbObj *)sdbGetRow(tsDbSdb, db); } -static int32_t mgmtCheckDBParams(SCMCreateDbMsg *pCreate) { - if (pCreate->commitLog < 0 || pCreate->commitLog > 2) { - mError("invalid db option commitLog: %d, only 0-2 allowed", pCreate->commitLog); - return TSDB_CODE_INVALID_OPTION; +static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) { + if (pCfg->maxCacheSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_BLOCK_SIZE) { + mError("invalid db option maxCacheSize:%d valid range: [%d, %d]", pCfg->maxCacheSize, TSDB_MIN_CACHE_BLOCK_SIZE, + TSDB_MAX_CACHE_BLOCK_SIZE); } - if (pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM) { - mError("invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, TSDB_REPLICA_MIN_NUM, - TSDB_REPLICA_MAX_NUM); - return TSDB_CODE_INVALID_OPTION; - } - - if (pCreate->daysPerFile < TSDB_FILE_MIN_PARTITION_RANGE || pCreate->daysPerFile > TSDB_FILE_MAX_PARTITION_RANGE) { - mError("invalid db option daysPerFile: %d valid range: [%d, %d]", pCreate->daysPerFile, TSDB_FILE_MIN_PARTITION_RANGE, - TSDB_FILE_MAX_PARTITION_RANGE); - return TSDB_CODE_INVALID_OPTION; - } - - if (pCreate->daysToKeep1 > pCreate->daysToKeep2 || pCreate->daysToKeep2 > pCreate->daysToKeep) { - mError("invalid db option daystokeep1: %d, daystokeep2: %d, daystokeep: %d", pCreate->daysToKeep1, - pCreate->daysToKeep2, pCreate->daysToKeep); - return TSDB_CODE_INVALID_OPTION; - } - - if (pCreate->daysToKeep1 < TSDB_FILE_MIN_PARTITION_RANGE || pCreate->daysToKeep1 < pCreate->daysPerFile) { - mError("invalid db option daystokeep: %d", pCreate->daysToKeep); - return TSDB_CODE_INVALID_OPTION; - } - - if (pCreate->rowsInFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCreate->rowsInFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { - mError("invalid db option rowsInFileBlock: %d valid range: [%d, %d]", pCreate->rowsInFileBlock, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); - return TSDB_CODE_INVALID_OPTION; - } - - if (pCreate->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCreate->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { - mError("invalid db option cacheBlockSize: %d valid range: [%d, %d]", pCreate->cacheBlockSize, - TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); - return TSDB_CODE_INVALID_OPTION; - } - - if (pCreate->maxSessions < TSDB_MIN_TABLES_PER_VNODE || pCreate->maxSessions > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option maxSessions: %d valid range: [%d, %d]", pCreate->maxSessions, TSDB_MIN_TABLES_PER_VNODE, + if (pCfg->maxTables < TSDB_MIN_TABLES_PER_VNODE || pCfg->maxTables > TSDB_MAX_TABLES_PER_VNODE) { + mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO) { - mError("invalid db option timePrecision: %d valid value: [%d, %d]", pCreate->precision, TSDB_TIME_PRECISION_MILLI, - TSDB_TIME_PRECISION_MICRO); + if (pCfg->daysPerFile < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysPerFile > TSDB_FILE_MAX_PARTITION_RANGE) { + mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_FILE_MIN_PARTITION_RANGE, + TSDB_FILE_MAX_PARTITION_RANGE); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->cacheNumOfBlocks.fraction < TSDB_MIN_AVG_BLOCKS || pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS) { - mError("invalid db option ablocks: %f valid value: [%d, %d]", pCreate->cacheNumOfBlocks.fraction, 0, TSDB_MAX_AVG_BLOCKS); + if (pCfg->daysToKeep1 < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysToKeep1 < pCfg->daysPerFile) { + mError("invalid db option daystokeep:%d", pCfg->daysToKeep); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->commitTime < TSDB_MIN_COMMIT_TIME_INTERVAL || pCreate->commitTime > TSDB_MAX_COMMIT_TIME_INTERVAL) { - mError("invalid db option commitTime: %d valid range: [%d, %d]", pCreate->commitTime, TSDB_MIN_COMMIT_TIME_INTERVAL, + if (pCfg->daysToKeep2 > pCfg->daysToKeep || pCfg->daysToKeep2 < pCfg->daysToKeep1) { + mError("invalid db option daystokeep1:%d, daystokeep2:%d, daystokeep:%d", pCfg->daysToKeep1, + pCfg->daysToKeep2, pCfg->daysToKeep); + return TSDB_CODE_INVALID_OPTION; + } + + if (pCfg->minRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { + mError("invalid db option minRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->minRowsPerFileBlock, + TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + return TSDB_CODE_INVALID_OPTION; + } + + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { + mError("invalid db option maxRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->maxRowsPerFileBlock, + TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + return TSDB_CODE_INVALID_OPTION; + } + + if (pCfg->maxRowsPerFileBlock < pCfg->minRowsPerFileBlock) { + mError("invalid db option minRowsPerFileBlock:%d maxRowsPerFileBlock:%d", pCfg->minRowsPerFileBlock, + pCfg->maxRowsPerFileBlock); + return TSDB_CODE_INVALID_OPTION; + } + + if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME_INTERVAL || pCfg->commitTime > TSDB_MAX_COMMIT_TIME_INTERVAL) { + mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME_INTERVAL, TSDB_MAX_COMMIT_TIME_INTERVAL); return TSDB_CODE_INVALID_OPTION; } - if (pCreate->compression < TSDB_MIN_COMPRESSION_LEVEL || pCreate->compression > TSDB_MAX_COMPRESSION_LEVEL) { - mError("invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, TSDB_MIN_COMPRESSION_LEVEL, + if (pCfg->precision != TSDB_TIME_PRECISION_MILLI && pCfg->precision != TSDB_TIME_PRECISION_MICRO) { + mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_TIME_PRECISION_MILLI, + TSDB_TIME_PRECISION_MICRO); + return TSDB_CODE_INVALID_OPTION; + } + + if (pCfg->compression < TSDB_MIN_COMPRESSION_LEVEL || pCfg->compression > TSDB_MAX_COMPRESSION_LEVEL) { + mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMPRESSION_LEVEL, TSDB_MAX_COMPRESSION_LEVEL); return TSDB_CODE_INVALID_OPTION; } + if (pCfg->commitLog < 0 || pCfg->commitLog > 2) { + mError("invalid db option commitLog:%d, only 0-2 allowed", pCfg->commitLog); + return TSDB_CODE_INVALID_OPTION; + } + + if (pCfg->replications < TSDB_REPLICA_MIN_NUM || pCfg->replications > TSDB_REPLICA_MAX_NUM) { + mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_REPLICA_MIN_NUM, + TSDB_REPLICA_MAX_NUM); + return TSDB_CODE_INVALID_OPTION; + } + return TSDB_CODE_SUCCESS; } -static int32_t mgmtCheckDbParams(SCMCreateDbMsg *pCreate) { - // assign default parameters - if (pCreate->maxSessions < 0) pCreate->maxSessions = tsSessionsPerVnode; - if (pCreate->cacheBlockSize < 0) pCreate->cacheBlockSize = tsCacheBlockSize; - if (pCreate->daysPerFile < 0) pCreate->daysPerFile = tsDaysPerFile; - if (pCreate->daysToKeep < 0) pCreate->daysToKeep = tsDaysToKeep; - if (pCreate->daysToKeep1 < 0) pCreate->daysToKeep1 = pCreate->daysToKeep; - if (pCreate->daysToKeep2 < 0) pCreate->daysToKeep2 = pCreate->daysToKeep; - if (pCreate->commitTime < 0) pCreate->commitTime = tsCommitTime; - if (pCreate->compression < 0) pCreate->compression = tsCompression; - if (pCreate->commitLog < 0) pCreate->commitLog = tsCommitLog; - if (pCreate->replications < 0) pCreate->replications = tsReplications; // - if (pCreate->rowsInFileBlock < 0) pCreate->rowsInFileBlock = tsRowsInFileBlock; // - if (pCreate->cacheNumOfBlocks.fraction < 0) pCreate->cacheNumOfBlocks.fraction = tsAverageCacheBlocks; // - - if (mgmtCheckDBParams(pCreate) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_INVALID_OPTION; - } - - pCreate->cacheNumOfBlocks.totalBlocks = (int32_t)(pCreate->cacheNumOfBlocks.fraction * pCreate->maxSessions); - - if (pCreate->cacheNumOfBlocks.totalBlocks > TSDB_MAX_CACHE_BLOCKS) { - mTrace("invalid db option cacheNumOfBlocks: %d valid range: [%d, %d]", pCreate->cacheNumOfBlocks.totalBlocks, - TSDB_MIN_CACHE_BLOCKS, TSDB_MAX_CACHE_BLOCKS); - return TSDB_CODE_INVALID_OPTION; - } - - // calculate the blocks per table - if (pCreate->blocksPerTable < 0) { - pCreate->blocksPerTable = pCreate->cacheNumOfBlocks.totalBlocks / 4; - } - - if (pCreate->blocksPerTable > pCreate->cacheNumOfBlocks.totalBlocks * 3 / 4) { - pCreate->blocksPerTable = pCreate->cacheNumOfBlocks.totalBlocks * 3 / 4; - } - - if (pCreate->blocksPerTable < TSDB_MIN_AVG_BLOCKS) { - pCreate->blocksPerTable = TSDB_MIN_AVG_BLOCKS; - } - - return TSDB_CODE_SUCCESS; +static void mgmtSetDefaultDbCfg(SDbCfg *pCfg) { + if (pCfg->maxCacheSize < 0) pCfg->maxCacheSize = tsMaxCacheSize; + if (pCfg->maxTables < 0) pCfg->maxTables = tsSessionsPerVnode; + if (pCfg->daysPerFile < 0) pCfg->daysPerFile = tsDaysPerFile; + if (pCfg->daysToKeep < 0) pCfg->daysToKeep = tsDaysToKeep; + if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep; + if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep; + if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsRowsInFileBlock; + if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = pCfg->minRowsPerFileBlock * 2; + if (pCfg->commitTime < 0) pCfg->commitTime = tsCommitTime; + if (pCfg->precision < 0) pCfg->precision = tsTimePrecision; + if (pCfg->compression < 0) pCfg->compression = tsCompression; + if (pCfg->commitLog < 0) pCfg->commitLog = tsCommitLog; + if (pCfg->replications < 0) pCfg->replications = tsReplications; } static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { int32_t code = acctCheck(pAcct, ACCT_GRANT_DB); - if (code != 0) { - return code; - } + if (code != 0) return code; SDbObj *pDb = mgmtGetDb(pCreate->db); if (pDb != NULL) { - mgmtDecDbRef(pDb); - + mgmtDecDbRef(pDb); if (pCreate->ignoreExist) { return TSDB_CODE_SUCCESS; } else { @@ -309,50 +285,41 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { } } - code = mgmtCheckDbParams(pCreate); - if (code != TSDB_CODE_SUCCESS) return code; - - assert(pCreate->daysToKeep1 <= pCreate->daysToKeep2 && pCreate->daysToKeep2 <= pCreate->daysToKeep); - code = grantCheck(TSDB_GRANT_DB); - if (code != 0) { + if (code != 0) return code; + + pDb = calloc(1, sizeof(SDbObj)); + strncpy(pDb->name, pCreate->db, TSDB_DB_NAME_LEN); + strncpy(pDb->acct, pAcct->user, TSDB_USER_LEN); + pDb->createdTime = taosGetTimestampMs(); + pDb->cfg = (SDbCfg) { + .maxCacheSize = 64,//(int64_t)pCreate->cacheBlockSize * pCreate->cacheNumOfBlocks.totalBlocks, + .maxTables = pCreate->maxSessions, + .daysPerFile = pCreate->daysPerFile, + .daysToKeep = pCreate->daysToKeep, + .daysToKeep1 = pCreate->daysToKeep1, + .daysToKeep2 = pCreate->daysToKeep2, + .minRowsPerFileBlock = pCreate->rowsInFileBlock * 1, + .maxRowsPerFileBlock = pCreate->rowsInFileBlock * 2, + .commitTime = pCreate->commitTime, + .precision = pCreate->precision, + .compression = pCreate->compression, + .commitLog = pCreate->commitLog, + .replications = pCreate->replications + }; + + mgmtSetDefaultDbCfg(&pDb->cfg); + + code = mgmtCheckDbCfg(&pDb->cfg); + if (code != TSDB_CODE_SUCCESS) { + tfree(pDb); return code; } - pDb = calloc(1, sizeof(SDbObj)); - - strncpy(pDb->name, pCreate->db, TSDB_DB_NAME_LEN); - strncpy(pCreate->acct, pAcct->user, TSDB_USER_LEN); - - pDb->createdTime = taosGetTimestampMs(); - - pDb->cfg = (SDbCfg) { - .vgId = pCreate->vgId, - .precision = pCreate->precision, - .maxSessions = pCreate->maxSessions, - .cacheNumOfBlocks.totalBlocks = pCreate->cacheNumOfBlocks.totalBlocks, - .rowsInFileBlock = pCreate->rowsInFileBlock, - .commitLog = pCreate->commitLog, - .replications = pCreate->replications, - .daysPerFile = pCreate->daysPerFile, - .cacheBlockSize = pCreate->cacheBlockSize, - .commitTime = pCreate->commitTime, - .blocksPerTable = pCreate->blocksPerTable, - .compression = pCreate->compression, - .daysToKeep = pCreate->daysToKeep, - .daysToKeep1 = pCreate->daysToKeep1, - .daysToKeep2 = pCreate->daysToKeep2, - .loadLatest = pCreate->loadLatest, - .repStrategy = pCreate->repStrategy, - }; - - strncpy(pDb->cfg.db, pCreate->db, TSDB_DB_NAME_LEN); - strncpy(pDb->cfg.acct, pCreate->acct, TSDB_USER_LEN); - SSdbOper oper = { - .type = SDB_OPER_GLOBAL, - .table = tsDbSdb, - .pObj = pDb, + .type = SDB_OPER_GLOBAL, + .table = tsDbSdb, + .pObj = pDb, .rowSize = sizeof(SDbObj), }; @@ -498,19 +465,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cache(b)"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "ablocks"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "tblocks"); + strcpy(pSchema[cols].name, "cache(Mb)"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -625,27 +580,15 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * if (strcmp(pUser->user, "root") == 0) { #endif pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxSessions; // table num can be created should minus 1 + *(int32_t *)pWrite = pDb->cfg.maxTables; // table num can be created should minus 1 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.rowsInFileBlock; + *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; -#ifdef _TD_ARM_32_ - *(int32_t *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); -#else - *(float *)pWrite = (pDb->cfg.cacheNumOfBlocks.totalBlocks * 1.0 / (pDb->cfg.maxSessions)); -#endif - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDb->cfg.blocksPerTable; + *(int32_t *)pWrite = pDb->cfg.maxCacheSize; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -747,7 +690,7 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { SDbCfg newCfg = pDb->cfg; int32_t daysToKeep = htonl(pAlter->daysToKeep); - int32_t maxSessions = htonl(pAlter->maxSessions); + int32_t maxTables = htonl(pAlter->maxSessions); int8_t replications = pAlter->replications; terrno = TSDB_CODE_SUCCESS; @@ -755,25 +698,28 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) { mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep); newCfg.daysToKeep = daysToKeep; - } else if (replications > 0 && replications != pDb->cfg.replications) { + } + + if (replications > 0 && replications != pDb->cfg.replications) { mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); terrno = TSDB_CODE_INVALID_OPTION; } newCfg.replications = replications; - } else if (maxSessions > 0 && maxSessions != pDb->cfg.maxSessions) { - mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxSessions, maxSessions); - if (maxSessions < TSDB_MIN_TABLES_PER_VNODE || maxSessions > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option tables: %d valid range: %d--%d", maxSessions, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + } + + if (maxTables > 0 && maxTables != pDb->cfg.maxTables) { + mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables); + if (maxTables < TSDB_MIN_TABLES_PER_VNODE || maxTables > TSDB_MAX_TABLES_PER_VNODE) { + mError("invalid db option tables: %d valid range: %d--%d", maxTables, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); terrno = TSDB_CODE_INVALID_OPTION; } - if (maxSessions < pDb->cfg.maxSessions) { - mError("invalid db option tables: %d should larger than original:%d", maxSessions, pDb->cfg.maxSessions); + if (maxTables < pDb->cfg.maxTables) { + mError("invalid db option tables: %d should larger than original:%d", maxTables, pDb->cfg.maxTables); terrno = TSDB_CODE_INVALID_OPTION; } - newCfg.maxSessions = maxSessions; - } else { + newCfg.maxTables = maxTables; } return newCfg; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 45db300e5f..0edb119724 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -106,9 +106,9 @@ static int32_t mgmtChildTableActionInsert(SSdbOper *pOper) { } mgmtDecDbRef(pDb); - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct == NULL) { - mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->cfg.acct); + mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->acct); return TSDB_CODE_INVALID_ACCT; } mgmtDecAcctRef(pAcct); @@ -148,9 +148,9 @@ static int32_t mgmtChildTableActionDelete(SSdbOper *pOper) { } mgmtDecDbRef(pDb); - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct == NULL) { - mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->cfg.acct); + mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->acct); return TSDB_CODE_INVALID_ACCT; } mgmtDecAcctRef(pAcct); @@ -929,7 +929,7 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc pStable->numOfColumns += ncols; pStable->sversion++; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries += (ncols * pStable->numOfTables); mgmtDecAcctRef(pAcct); @@ -966,7 +966,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize); - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries -= pStable->numOfTables; mgmtDecAcctRef(pAcct); @@ -1505,7 +1505,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc pTable->numOfColumns += ncols; pTable->sversion++; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries += ncols; mgmtDecAcctRef(pAcct); @@ -1539,7 +1539,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch pTable->numOfColumns--; pTable->sversion++; - SAcctObj *pAcct = mgmtGetAcct(pDb->cfg.acct); + SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries--; mgmtDecAcctRef(pAcct); diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index bec4dbe83a..e053ea66f5 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -71,14 +71,14 @@ static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) { pVgroup->prev = NULL; pVgroup->next = NULL; - int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions; - pVgroup->tableList = calloc(pDb->cfg.maxSessions, sizeof(SChildTableObj *)); + int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables; + pVgroup->tableList = calloc(pDb->cfg.maxTables, sizeof(SChildTableObj *)); if (pVgroup->tableList == NULL) { mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size); return -1; } - pVgroup->idPool = taosInitIdPool(pDb->cfg.maxSessions); + pVgroup->idPool = taosInitIdPool(pDb->cfg.maxTables); if (pVgroup->idPool == NULL) { mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId); tfree(pVgroup->tableList); @@ -146,15 +146,15 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) { int32_t oldTables = taosIdPoolMaxSize(pVgroup->idPool); SDbObj *pDb = pVgroup->pDb; if (pDb != NULL) { - if (pDb->cfg.maxSessions != oldTables) { - mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxSessions); - taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxSessions); - int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxSessions; + if (pDb->cfg.maxTables != oldTables) { + mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables); + taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables); + int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables; pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size); } } - mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxSessions, pVgroup->numOfVnodes); + mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxTables, pVgroup->numOfVnodes); return TSDB_CODE_SUCCESS; } @@ -511,7 +511,7 @@ void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { pVgroup->numOfTables++; } - if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables) mgmtAddVgroupIntoDbTail(pVgroup); } @@ -522,7 +522,7 @@ void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { pVgroup->numOfTables--; } - if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxSessions) + if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables) mgmtAddVgroupIntoDbTail(pVgroup); } @@ -535,16 +535,16 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SMDVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); - pCfg->maxTables = htonl(pDb->cfg.maxSessions); - pCfg->maxCacheSize = htobe64((int64_t)pDb->cfg.cacheBlockSize * pDb->cfg.cacheNumOfBlocks.totalBlocks); - pCfg->maxCacheSize = htobe64(-1); - pCfg->minRowsPerFileBlock = htonl(-1); - pCfg->maxRowsPerFileBlock = htonl(-1); + pCfg->maxTables = htonl(pDb->cfg.maxTables); + pCfg->maxCacheSize = htobe64(pDb->cfg.maxCacheSize); + pCfg->maxCacheSize = htobe64(-1); //TODO + pCfg->minRowsPerFileBlock = htonl(-1); //TODO + pCfg->maxRowsPerFileBlock = htonl(-1); //TODO pCfg->daysPerFile = htonl(pDb->cfg.daysPerFile); pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); pCfg->daysToKeep = htonl(pDb->cfg.daysToKeep); - pCfg->daysToKeep = htonl(-1); + pCfg->daysToKeep = htonl(-1); //TODO pCfg->commitTime = htonl(pDb->cfg.commitTime); pCfg->precision = pDb->cfg.precision; pCfg->compression = pDb->cfg.compression; From 46a7c3c91291086bb5048797d8bfdd67b9886000 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 24 Apr 2020 23:54:39 +0800 Subject: [PATCH 42/63] add insert/int.py and insert/float.py to test insert function. [TD-167] --- tests/pytest/simpletest.sh | 6 ++++++ tests/pytest/test.py | 7 +++++-- tests/pytest/util/cases.py | 8 ++++++-- tests/pytest/util/dnodes.py | 32 +++++++++++++++++++++----------- tests/pytest/util/sql.py | 2 +- 5 files changed, 39 insertions(+), 16 deletions(-) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh index b77c1aa142..58345e13ee 100755 --- a/tests/pytest/simpletest.sh +++ b/tests/pytest/simpletest.sh @@ -1,3 +1,9 @@ #!/bin/bash python3 ./test.py -f insert/basic.py $1 python3 ./test.py -s $1 +sleep 1 +python3 ./test.py -f insert/int.py $1 +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py -f insert/float.py $1 +python3 ./test.py -s $1 diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 55ec46e526..ae82ed045c 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -100,12 +100,15 @@ if __name__ == "__main__": tdDnodes.deploy(1) tdDnodes.start(1) conn = taos.connect( - host='192.168.0.1', + host='127.0.0.1', config=tdDnodes.getSimCfgPath()) if fileName == "all": tdCases.runAllLinux(conn) else: - tdCases.runOneLinux(conn, fileName) + try: + tdCases.runOneLinux(conn, fileName) + except Exception as e: + tdLog.exit("failed: %s" % fileName) conn.close() else: tdLog.notice("Procedures for tdengine deployed in %s" % (masterIp)) diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 4bd1b1e9d1..dd1f9e1649 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -67,12 +67,16 @@ class TDCases: if tmp.name.find(fileName) != -1: case = testModule.TDTestCase() case.init(conn) - case.run() + try: + case.run() + except Exception as e: + tdLog.notice(repr(e)) + tdLog.exit("failed: %s" % fileName) case.stop() runNum += 1 continue - tdLog.notice("total %d Linux test case(s) executed" % (runNum)) + tdLog.success("total %d Linux test case(s) executed" % (runNum)) def runAllWindows(self, conn): # TODO: load all Windows cases here diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 6a9c2607e6..51eed3d25a 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -19,12 +19,19 @@ from util.log import * class TDSimClient: + def __init__(self): + self.testCluster = False + def init(self, path): + self.__init__() self.path = path def getCfgDir(self): return self.cfgDir + def setTestCluster(self, value): + self.testCluster = value + def cfg(self, option, value): cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: @@ -55,8 +62,9 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") self.cfg("logDir", self.logDir) self.cfg("numOfLogLines", "100000000") self.cfg("numOfThreadsPerCore", "2.0") @@ -128,11 +136,12 @@ class TDDnode: if self.testCluster: self.startIP() - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("publicIp", "192.168.0.%d" % (self.index)) + self.cfg("internalIp", "192.168.0.%d" % (self.index)) + self.cfg("privateIp", "192.168.0.%d" % (self.index)) self.cfg("dataDir", self.dataDir) self.cfg("logDir", self.logDir) self.cfg("numOfLogLines", "100000000") @@ -291,10 +300,6 @@ class TDDnodes: for i in range(len(self.dnodes)): self.dnodes[i].init(self.path) - self.sim = TDSimClient() - self.sim.init(self.path) - self.sim.deploy() - def setTestCluster(self, value): self.testCluster = value @@ -302,6 +307,11 @@ class TDDnodes: self.valgrind = value def deploy(self, index): + self.sim = TDSimClient() + self.sim.init(self.path) + self.sim.setTestCluster(self.testCluster) + self.sim.deploy() + self.check(index) self.dnodes[index - 1].setTestCluster(self.testCluster) self.dnodes[index - 1].setValgrind(self.valgrind) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index b4ac845bc8..0008468130 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -63,7 +63,7 @@ class TDSql: def checkRows(self, expectRows): if self.queryRows != expectRows: tdLog.exit( - "sql:%.40s, queryRows:%d != expect:%d" % + "failed: sql:%.40s, queryRows:%d != expect:%d" % (self.sql, self.queryRows, expectRows)) tdLog.info("sql:%.40s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows)) From c4eed9ec6959916cb315bbffd788d53673a178b9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 25 Apr 2020 00:03:41 +0800 Subject: [PATCH 43/63] expose more info if case failed. --- .travis.yml | 2 +- tests/pytest/insert/float.py | 147 ++++++++++++++++++++++++++++ tests/pytest/insert/int.py | 184 +++++++++++++++++++++++++++++++++++ tests/pytest/test.py | 5 +- tests/pytest/util/cases.py | 2 +- tests/test-all.sh | 6 +- 6 files changed, 338 insertions(+), 8 deletions(-) create mode 100644 tests/pytest/insert/float.py create mode 100644 tests/pytest/insert/int.py diff --git a/.travis.yml b/.travis.yml index db6ce79703..39fddc20c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ matrix: memError=`grep -m 1 'ERROR SUMMARY' mem-error-out.txt | awk '{print $4}'` if [ -n "$memError" ]; then - if [ "$memError" -gt 16 ] && [ "$defiMemError" -gt 0 ]; then + if [ "$memError" -gt 16 ] || [ "$defiMemError" -gt 0 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" travis_terminate $memError diff --git a/tests/pytest/insert/float.py b/tests/pytest/insert/float.py new file mode 100644 index 0000000000..7c4474f7ad --- /dev/null +++ b/tests/pytest/insert/float.py @@ -0,0 +1,147 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import datetime + +import taos + +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info("=============== step1") + cmd = 'create table tb (ts timestamp, speed float)' + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = 'insert into tb values (now, -3.40E+38)' + tdLog.info(cmd) + tdSql.execute(cmd) + + tdLog.info("=============== step2") + cmd = 'insert into tb values (now+1a, 3.40E+308)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'select * from tb order by ts desc' + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(1) + + tdLog.info("=============== step3") + cmd = "insert into tb values (now+2a, 2.85)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + ret = tdSql.query(cmd) + tdSql.checkRows(2) + + if ((abs(tdSql.getData(0, 1) - 2.850000)) > 1.0e-7): + tdLog.exit("data is not 2.850000") + + tdLog.info("=============== step4") + cmd = "insert into tb values (now+3a, 3.4)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(3) + if (abs(tdSql.getData(0, 1) - 3.400000) > 1.0e-7): + tdLog.exit("data is not 3.400000") + + tdLog.info("=============== step5") + cmd = "insert into tb values (now+4a, a2)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit("This test failed: \ + insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+4a, 0)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(4) + if (abs(tdSql.getData(0, 1) - 0.000000) != 0): + tdLog.exit("data is not 0.000000") + + tdLog.info("=============== step6") + cmd = "insert into tb values (now+5a, 2a)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+5a, 2)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + ret = tdSql.query(cmd) + tdSql.checkRows(5) + if (abs(tdSql.getData(0, 1) - 2.000000) > 1.0e-7): + tdLog.info("data is not 2.000000") + + tdLog.info("=============== step7") + cmd = "insert into tb values (now+6a, 2a'1)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+6a, 2)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + if (abs(tdSql.getData(0, 1) - 2.000000) > 1.0e-7): + tdLog.exit("data is not 2.000000") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/int.py b/tests/pytest/insert/int.py new file mode 100644 index 0000000000..8d38337576 --- /dev/null +++ b/tests/pytest/insert/int.py @@ -0,0 +1,184 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import datetime + +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info("=============== step1") + tdSql.execute('create table tb (ts timestamp, speed int)') + + cmd = 'insert into tb values (now, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(1) + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step2") + cmd = 'insert into tb values (now+1m, -2147483648)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: INT data overflow error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("INT data overflow error catched") + + cmd = 'insert into tb values (now+1m, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(2) + + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step3") + cmd = 'insert into tb values (now+2m, 2147483647)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(3) + if(tdSql.getData(0, 1) != 2147483647): + tdLog.exit("data is not 2147483647") + + tdLog.info("=============== step4") + cmd = 'insert into tb values (now+3m, 2147483648)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: INT data overflow error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("INT data overflow error catched") + + cmd = 'insert into tb values (now+3m, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(4) + + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step5") + cmd = 'insert into tb values (now+4m, a2)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+4m, 0)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(5) + + if(tdSql.getData(0, 1) != 0): + tdLog.exit("data is not 0") + + tdLog.info("=============== step6") + cmd = 'insert into tb values (now+5m, 2a)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+5m, 2)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(6) + if (tdSql.getData(0, 1) != 2): + tdLog.exit("data is not 2") + + tdLog.info("=============== step7") + cmd = "insert into tb values (now+6m, 2a'1)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+6m, 2)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(7) + if (tdSql.getData(0, 1) != 2): + tdLog.exit("data is not 2") + + tdLog.info("=============== step8") + cmd = 'insert into tb values (now+8m, "null")' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(8) + + if (tdSql.getData(0, 1) is not None): + tdLog.exit("data is not null") + + tdLog.info("=============== step9") + cmd = "insert into tb values (now+9m, 'null')" + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(9) + if (tdSql.getData(0, 1) is not None): + tdLog.exit("data is not null") + + tdLog.info("=============== step10") + cmd = 'insert into tb values (now+10m, -123)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(10) + + if (tdSql.getData(0, 1) != -123): + tdLog.exit("data is not -123") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index ae82ed045c..479406a00b 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -105,10 +105,7 @@ if __name__ == "__main__": if fileName == "all": tdCases.runAllLinux(conn) else: - try: - tdCases.runOneLinux(conn, fileName) - except Exception as e: - tdLog.exit("failed: %s" % fileName) + tdCases.runOneLinux(conn, fileName) conn.close() else: tdLog.notice("Procedures for tdengine deployed in %s" % (masterIp)) diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index dd1f9e1649..1771c23620 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -71,7 +71,7 @@ class TDCases: case.run() except Exception as e: tdLog.notice(repr(e)) - tdLog.exit("failed: %s" % fileName) + tdLog.notice("%s failed: %s" % (__file__, fileName)) case.stop() runNum += 1 continue diff --git a/tests/test-all.sh b/tests/test-all.sh index dee89b9dc5..6943dd47a7 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -25,15 +25,17 @@ if [ "$totalFailed" -ne "0" ]; then fi cd ../pytest -./simpletest.sh 2>&1 | grep 'successfully executed\|failed' | tee pytest-out.txt +./simpletest.sh 2>&1 | tee pytest-out.txt totalPySuccess=`grep 'successfully executed' pytest-out.txt | wc -l` if [ "$totalPySuccess" -gt "0" ]; then + grep 'successfully executed' pytest-out.txt echo -e "${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" fi -totalPyFailed=`grep 'failed' pytest-out.txt | wc -l` +totalPyFailed=`grep 'failed\|fault' pytest-out.txt | wc -l` if [ "$totalPyFailed" -ne "0" ]; then + cat pytest-out.txt echo -e "${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" exit $totalPyFailed fi From c597c8ad587f8926c42eee99bf3b2923475a5bdb Mon Sep 17 00:00:00 2001 From: localvar Date: Sat, 25 Apr 2020 10:02:16 +0800 Subject: [PATCH 44/63] TD-97: fix bug and update testcases --- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscServer.c | 2 +- src/query/src/qast.c | 3 ++- src/query/src/queryExecutor.c | 20 +++++++++++--------- src/tsdb/src/tsdbRead.c | 15 +++++++-------- tests/pytest/{select => query}/__init__.py | 0 tests/pytest/{select => query}/tbname.py | 17 ++++++++++------- 7 files changed, 32 insertions(+), 27 deletions(-) rename tests/pytest/{select => query}/__init__.py (100%) rename tests/pytest/{select => query}/tbname.py (90%) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index df47afd0dc..0e23236b16 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3545,7 +3545,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac return TSDB_CODE_SUCCESS; } - SStringBuilder sb1; + SStringBuilder sb1 = { 0 }; taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); char db[TSDB_TABLE_ID_LEN] = {0}; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index aeb137189b..b8c0274dd9 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1855,7 +1855,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->msgType = TSDB_MSG_TYPE_CM_HEARTBEAT; assert(msgLen + minMsgSize() <= size); - return msgLen; + return TSDB_CODE_SUCCESS; } int tscProcessTableMetaRsp(SSqlObj *pSql) { diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 70e4dbd803..b262429e5d 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -816,7 +816,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, if (pQueryInfo->optr == TSDB_RELATION_IN) { addToResult = pQueryInfo->compare(table->name, pQueryInfo->q.arr); } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) { - addToResult = pQueryInfo->compare(table->name, pQueryInfo->q.pz); + addToResult = !pQueryInfo->compare(table->name, pQueryInfo->q.pz); } } else { // TODO: other columns @@ -1188,6 +1188,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { // TODO: } memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN, len); + pVal->nLen = (int32_t)len; } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { right->nodeType = TSQL_NODE_VALUE; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index b9b304cfc1..cdc7b3328c 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5430,13 +5430,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; } - // the tag query condition expression string is located at the end of query msg - if (pQueryMsg->tagCondLen > 0) { - *tagCond = calloc(1, pQueryMsg->tagCondLen); - memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); - pMsg += pQueryMsg->tagCondLen; - } - if (pQueryMsg->numOfTags > 0) { (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { @@ -5448,13 +5441,22 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pTagCol->numOfFilters = 0; (*tagCols)[i] = *pTagCol; + pMsg += sizeof(SColumnInfo); } } + // the tag query condition expression string is located at the end of query msg + if (pQueryMsg->tagCondLen > 0) { + *tagCond = calloc(1, pQueryMsg->tagCondLen); + memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); + pMsg += pQueryMsg->tagCondLen; + } + if (*pMsg != 0) { - size_t len = strlen(pMsg); - *tbnameCond = malloc(len + 1); + size_t len = strlen(pMsg) + 1; + *tbnameCond = malloc(len); strcpy(*tbnameCond, pMsg); + pMsg += len; } qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 0449f42eb5..eb35be5383 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1448,7 +1448,7 @@ int32_t tsdbQueryByTagsCond( size_t len, int16_t tagNameRelType, const char* tbnameCond, - STableGroupInfo *pGroupList, + STableGroupInfo *pGroupInfo, SColIndex *pColIndex, int32_t numOfCols ) { @@ -1465,16 +1465,15 @@ int32_t tsdbQueryByTagsCond( if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { int32_t ret = getAllTableIdList(tsdb, uid, res); if (ret == TSDB_CODE_SUCCESS) { - pGroupList->numOfTables = taosArrayGetSize(res); - pGroupList->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + pGroupInfo->numOfTables = taosArrayGetSize(res); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); } - - pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); taosArrayDestroy(res); return ret; } + int32_t ret = TSDB_CODE_SUCCESS; + tExprNode* expr = exprTreeFromTableName(tbnameCond); tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); if (tagExpr != NULL) { @@ -1491,8 +1490,8 @@ int32_t tsdbQueryByTagsCond( } doQueryTableList(pSTable, res, expr); - pGroupList->numOfTables = taosArrayGetSize(res); - pGroupList->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + pGroupInfo->numOfTables = taosArrayGetSize(res); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); taosArrayDestroy(res); return ret; diff --git a/tests/pytest/select/__init__.py b/tests/pytest/query/__init__.py similarity index 100% rename from tests/pytest/select/__init__.py rename to tests/pytest/query/__init__.py diff --git a/tests/pytest/select/tbname.py b/tests/pytest/query/tbname.py similarity index 90% rename from tests/pytest/select/tbname.py rename to tests/pytest/query/tbname.py index f088f9e63a..d2799efa25 100644 --- a/tests/pytest/select/tbname.py +++ b/tests/pytest/query/tbname.py @@ -39,29 +39,32 @@ class TDTestCase: tdSql.query("select * from cars where tbname in ('carzero', 'cartwo')") tdSql.checkRows(1) - tdSql.query("select * from cars where id=1 or tbname in ('carzero0', 'cartwo')") + tdSql.query("select * from cars where id=1 or tbname in ('carzero', 'cartwo')") tdSql.checkRows(2) - tdSql.query("select * from cars where id=1 and tbname in ('carzero0', 'cartwo')") + tdSql.query("select * from cars where id=1 and tbname in ('carzero', 'cartwo')") tdSql.checkRows(0) - tdSql.query("select * from cars where id=0 and tbname in ('carzero0', 'cartwo')") + tdSql.query("select * from cars where id=0 and tbname in ('carzero', 'cartwo')") tdSql.checkRows(1) - tdSql.query("select * from cars where tbname like 'car%') + """ + tdSql.query("select * from cars where tbname like 'car%'") tdSql.checkRows(2) - tdSql.query("select * from cars where tbname like '%%o') + tdSql.cursor.execute("use db") + tdSql.query("select * from cars where tbname like '%o'") tdSql.checkRows(1) tdSql.query("select * from cars where id=1 and tbname like 'car%') tdSql.checkRows(1) - tdSql.query("select * from cars where id = 1 and tbname like '%%o') + tdSql.query("select * from cars where id = 1 and tbname like '%o') tdSql.checkRows(0) - tdSql.query("select * from cars where id = 1 or tbname like '%%o') + tdSql.query("select * from cars where id = 1 or tbname like '%o') tdSql.checkRows(2) + """ def stop(self): tdSql.close() From 886d8d58bd0f7e2a5e07865807b183c7897b5052 Mon Sep 17 00:00:00 2001 From: localvar Date: Sat, 25 Apr 2020 10:35:29 +0800 Subject: [PATCH 45/63] TD-97: remove duplicated definition --- src/inc/taosdef.h | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 15fb026381..b476c866d9 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -32,19 +32,8 @@ extern "C" { #define TSKEY int64_t #endif -// Data type definition -#define TSDB_DATA_TYPE_NULL 0 // 1 bytes -#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes -#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte -#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes -#define TSDB_DATA_TYPE_INT 4 // 4 bytes -#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes -#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes -#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string -#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes -#define TSDB_DATA_TYPE_NCHAR 10 // unicode string -#define TSDB_DATA_TYPE_ARRAY 11 // only used 'in' query to hold the values +// this data type is internally used only in 'in' query to hold the values +#define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) // Bytes for each type. extern const int32_t TYPE_BYTES[11]; From 25e13520708acd2535ab1139321b846fdf175eba Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 25 Apr 2020 13:52:39 +0800 Subject: [PATCH 46/63] [td-187] pattern match test case. --- src/query/tests/patternMatchTest.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index 2e70f26269..41156ce8ff 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -58,7 +58,7 @@ TEST(testCase, patternMatchTest) { EXPECT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); str = "abcdefgabcdeju"; - ret = patternMatch("abc%f_", str, 1, &info); + ret = patternMatch("abc%f_", str, 1, &info); // pattern string is longe than the size EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); str = "abcdefgabcdeju"; @@ -72,4 +72,8 @@ TEST(testCase, patternMatchTest) { str = "abcdefgabcdeju"; ret = patternMatch("a__", str, 2, &info); EXPECT_EQ(ret, TSDB_PATTERN_NOMATCH); + + str = "carzero"; + ret = patternMatch("%o", str, strlen(str), &info); + EXPECT_EQ(ret, TSDB_PATTERN_MATCH); } From b345312f8b9dba411ec63548c6779e3160aa5a54 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 25 Apr 2020 16:20:18 +0800 Subject: [PATCH 47/63] [TD-147] --- src/inc/taosmsg.h | 1 - src/mnode/src/mgmtTable.c | 22 ++++++++++++++++++++-- src/vnode/src/vnodeWrite.c | 5 ++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index dd6dc52242..5f3ea93afb 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -345,7 +345,6 @@ typedef struct { } SMDDropTableMsg; typedef struct { - int32_t contLen; int32_t vgId; int64_t uid; char tableId[TSDB_TABLE_ID_LEN + 1]; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 0edb119724..1813be6522 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -756,8 +756,26 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) { static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) { SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; if (pStable->numOfTables != 0) { - mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables); - mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); + mgmtDropAllChildTablesInStable(pStable); + for (int32_t vg = 0; vg < pTable->vgLen; ++vg) { + int32_t vgId = pTable->vgList[vg]; + if (vgId == 0) break; + + SMDDropSTableMsg *pDrop = rpcMalloc(sizeof(SMDDropSTableMsg)); + pDrop->vgId = htonl(vgId); + pDrop->uid = htobe64(pStable->uid); + mgmtExtractTableName(pStable->info.tableId, pDrop->tableId); + + SVgObj *pVgroup = mgmtGetVgroup(vgId); + if (pVgroup != NULL) { + SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup); + SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; + mgmtSendMsgToDnode(&ipSet, &rpcMsg); + mgmtDecVgroupRef(pVgroup); + } + } + //mError("stable:%s, numOfTables:%d not 0", pStable->info.tableId, pStable->numOfTables); + //mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_OTHERS); } else { SSdbOper oper = { .type = SDB_OPER_GLOBAL, diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 81cba7b6fa..26808f2e4e 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -224,11 +224,10 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet int32_t code = 0; dTrace("pVnode:%p vgId:%d, stable:%s, start to drop", pVnode, pVnode->vgId, pTable->tableId); - // int64_t uid = htobe64(pTable->uid); - // TODO: drop stable in vvnode + //int64_t uid = htobe64(pTable->uid); //void *pTsdb = dnodeGetVnodeTsdb(pMsg->pVnode); - //rpcRsp.code = tsdbDropSTable(pTsdb, pTable->uid); + //rpcRsp.code = tsdbDropTable(pTsdb, pTable->uid); code = TSDB_CODE_SUCCESS; dTrace("pVnode:%p vgId:%d, stable:%s, drop stable result:%x", pVnode, pTable->tableId, code); From ba6acfd4988596fec0361b34d751d8cb9d23944c Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 25 Apr 2020 23:01:13 +0800 Subject: [PATCH 48/63] [TD-197] fix bugs in arithmetic expression calculation --- src/client/inc/tsclient.h | 2 +- src/client/src/tscFunctionImpl.c | 22 +-- src/client/src/tscSQLParser.c | 132 +++++++------- src/client/src/tscServer.c | 6 +- src/client/src/tscSql.c | 43 +++-- src/client/src/tscSubquery.c | 4 +- src/client/src/tscSystem.c | 7 +- src/client/src/tscUtil.c | 7 +- src/inc/taosmsg.h | 28 +-- src/query/inc/qast.h | 7 +- src/query/inc/tsqlfunction.h | 4 +- src/query/inc/tvariant.h | 2 +- src/query/src/qast.c | 31 +--- src/query/src/queryExecutor.c | 289 ++++++++++++++++--------------- src/query/src/tvariant.c | 10 +- 15 files changed, 292 insertions(+), 302 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index d6fb9279c7..e8278ea145 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -78,7 +78,7 @@ typedef struct STableMetaInfo { */ int32_t vgroupIndex; char name[TSDB_TABLE_ID_LEN]; // (super) table name - SArray* tagColList; // involved tag columns + SArray* tagColList; // SArray, involved tag columns } STableMetaInfo; /* the structure for sql function in select clause */ diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 92bd535d41..bc6231ba09 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3294,29 +3294,26 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -char *arithmetic_callback_function(void *param, char *name, int32_t colId) { +char *getArithColumnData(void *param, const char* name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - SArithExprInfo *pExpr = pSupport->pArithExpr; - int32_t colIndex = -1; - - for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { - if (colId == pExpr->binExprInfo.pReqColumns[i].colId) { - colIndex = pExpr->binExprInfo.pReqColumns[i].colIndex; + int32_t index = -1; + for (int32_t i = 0; i < pSupport->numOfCols; ++i) { + if (colId == pSupport->colList[i].colId) { + index = i; break; } } - assert(colIndex >= 0 && colId >= 0); - return pSupport->data[colIndex] + pSupport->offset * pSupport->elemSize[colIndex]; + assert(index >= 0 && colId >= 0); + return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - tSQLBinaryExprCalcTraverse(sas->pArithExpr->binExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, - arithmetic_callback_function); + tExprTreeCalcTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; pCtx->param[1].pz = NULL; @@ -3327,8 +3324,7 @@ static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; sas->offset = index; - tSQLBinaryExprCalcTraverse(sas->pArithExpr->binExprInfo.pBinExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, - arithmetic_callback_function); + tExprTreeCalcTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); pCtx->aOutputBuf += pCtx->outputBytes; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 59e2f147e1..8a27241986 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -101,7 +101,7 @@ static void updateTagColumnIndex(SQueryInfo* pQueryInfo, int32_t tableIndex); static int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); -static int32_t getColumnIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getColumnIndexByName(const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t optrToString(tSQLExpr* pExpr, char** exprString); @@ -116,7 +116,7 @@ static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSql static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); -static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); +static int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); /* * Used during parsing query sql. Since the query sql usually small in length, error position @@ -1171,13 +1171,32 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel SColumnIndex index = {.tableIndex = tableIndex}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), sizeof(double), false); - addExprParams(pExpr, arithmeticExprStr, TSDB_DATA_TYPE_BINARY, strlen(arithmeticExprStr), index.tableIndex); - + /* todo alias name should use the original sql string */ char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; strncpy(pExpr->aliasName, name, TSDB_COL_NAME_LEN); - + + tExprNode* pNode = NULL; + SArray* colList = taosArrayInit(10, sizeof(SColIndex)); + + int32_t ret = exprTreeFromSqlExpr(&pNode, pItem->pNode, pQueryInfo->exprsInfo, pQueryInfo, colList); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(&pNode, NULL); + return invalidSqlErrMsg(pQueryInfo->msg, "invalid arithmetic expression in select clause"); + } + + SBuffer buf = exprTreeToBinary(pNode); + + size_t len = tbufTell(&buf); + char* c = tbufGetData(&buf, true); + + // set the serialized binary string as the parameter of arithmetic expression + addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, len, index.tableIndex); + insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); + + taosArrayDestroy(colList); + tExprTreeDestroy(&pNode, NULL); } else { columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; @@ -1196,8 +1215,6 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel pFuncExpr->interResBytes = sizeof(double); pFuncExpr->type = TSDB_DATA_TYPE_DOUBLE; - SExprInfo* pBinExprInfo = &pFuncExpr->binExprInfo; - tExprNode* pNode = NULL; // SArray* colList = taosArrayInit(10, sizeof(SColIndex)); @@ -1206,26 +1223,26 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(pQueryInfo->msg, "invalid expression in select clause"); } - - pBinExprInfo->pBinExpr = pNode; + + pFuncExpr->pExpr = pNode; assert(0); -// pBinExprInfo->pReqColumns = pColIndex; +// pExprInfo->pReqColumns = pColIndex; - for(int32_t k = 0; k < pBinExprInfo->numOfCols; ++k) { - SColIndex* pCol = &pBinExprInfo->pReqColumns[k]; - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - - for(int32_t f = 0; f < size; ++f) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, f); - if (strcmp(pExpr->aliasName, pCol->name) == 0) { - pCol->colIndex = f; - break; - } - } - - assert(pCol->colIndex >= 0 && pCol->colIndex < size); - tfree(pNode); - } +// for(int32_t k = 0; k < pFuncExpr->numOfCols; ++k) { +// SColIndex* pCol = &pFuncExpr->colList[k]; +// size_t size = tscSqlExprNumOfExprs(pQueryInfo); +// +// for(int32_t f = 0; f < size; ++f) { +// SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, f); +// if (strcmp(pExpr->aliasName, pCol->name) == 0) { +// pCol->colIndex = f; +// break; +// } +// } +// +// assert(pCol->colIndex >= 0 && pCol->colIndex < size); +// tfree(pNode); +// } } } } else { @@ -1317,28 +1334,6 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c void addRequiredTagColumn(STableMetaInfo* pTableMetaInfo, SColumnIndex* index) { tscColumnListInsert(pTableMetaInfo->tagColList, index); -// if (pTableMetaInfo->numOfTags == 0 || pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags - 1] < tagColIndex) { -// pTableMetaInfo->tagColumnIndex[pTableMetaInfo->numOfTags++] = tagColIndex; -// } else { // find the appropriate position -// for (int32_t i = 0; i < pTableMetaInfo->numOfTags; ++i) { -// if (tagColIndex > pTableMetaInfo->tagColumnIndex[i]) { -// continue; -// } else if (tagColIndex == pTableMetaInfo->tagColumnIndex[i]) { -// break; -// } else { -// memmove(&pTableMetaInfo->tagColumnIndex[i + 1], &pTableMetaInfo->tagColumnIndex[i], -// sizeof(pTableMetaInfo->tagColumnIndex[0]) * (pTableMetaInfo->numOfTags - i)); -// -// pTableMetaInfo->tagColumnIndex[i] = tagColIndex; -// -// pTableMetaInfo->numOfTags++; -// break; -// } -// } -// } - - // plus one means tbname -// assert(tagColIndex >= -1 && tagColIndex < TSDB_MAX_TAGS && pTableMetaInfo->numOfTags <= TSDB_MAX_TAGS + 1); } static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { @@ -2047,7 +2042,7 @@ int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIn return TSDB_CODE_SUCCESS; } -int32_t getColumnIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getColumnIndexByName(const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { return TSDB_CODE_INVALID_SQL; } @@ -3753,13 +3748,24 @@ static int32_t getTagQueryCondExpr(SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { tSQLExpr* p1 = extractExprForSTable(pExpr, pQueryInfo, i); tExprNode* p = NULL; - - ret = exprTreeFromSqlExpr(&p, p1, NULL, pQueryInfo, NULL); + + SArray* colList = taosArrayInit(10, sizeof(SColIndex)); + ret = exprTreeFromSqlExpr(&p, p1, NULL, pQueryInfo, colList); SBuffer buf = exprTreeToBinary(p); - int64_t uid = tscGetMetaInfo(pQueryInfo, i)->pTableMeta->uid; + // add to source column list + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + int64_t uid = pTableMetaInfo->pTableMeta->uid; + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + + size_t num = taosArrayGetSize(colList); + for(int32_t j = 0; j < num; ++j) { + SColIndex* pIndex = taosArrayGet(colList, j); + SColumnIndex index = {.tableIndex = i, .columnIndex = pIndex->colIndex - numOfCols}; + addRequiredTagColumn(pTableMetaInfo, &index); + } + tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &buf); - doCompactQueryExpr(pExpr); tSQLExprDestroy(p1); @@ -5871,7 +5877,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_SUCCESS; // Does not build query message here } -int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { +int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; @@ -5896,8 +5902,9 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SArray* pExpr (*pExpr)->pVal = calloc(1, sizeof(tVariant)); tVariantAssign((*pExpr)->pVal, &pSqlExpr->val); - + return TSDB_CODE_SUCCESS; } else if (pSqlExpr->nSQLOptr >= TK_COUNT && pSqlExpr->nSQLOptr <= TK_AVG_IRATE) { + // arithmetic expression on the results of aggregation functions *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_COL; (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); @@ -5915,7 +5922,7 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SArray* pExpr break; } } - } else if (pSqlExpr->nSQLOptr == TK_ID) { // column name + } else if (pSqlExpr->nSQLOptr == TK_ID) { // column name, normal column arithmetic expression SColumnIndex index = {0}; int32_t ret = getColumnIndexByName(&pSqlExpr->colInfo, pQueryInfo, &index); if (ret != TSDB_CODE_SUCCESS) { @@ -5929,18 +5936,21 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, tSQLExpr* pSqlExpr, SArray* pExpr STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); *(*pExpr)->pSchema = *pSchema; + + if (pCols != NULL) { // record the involved columns + SColIndex colIndex = {0}; + strncpy(colIndex.name, pSchema->name, TSDB_COL_NAME_LEN); + colIndex.colId = pSchema->colId; + colIndex.colIndex = index.columnIndex; + + taosArrayPush(pCols, &colIndex); + } return TSDB_CODE_SUCCESS; } else { return TSDB_CODE_INVALID_SQL; } - if (pCols != NULL) { // record the involved columns - SColIndex colIndex = {0}; - strncpy(colIndex.name, pSqlExpr->operand.z, pSqlExpr->operand.n); - taosArrayPush(pCols, &colIndex); - } - } else { *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_EXPR; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f6d20f69ac..af623596a4 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -788,9 +788,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen); - - // by plus one char to make the string null-terminated - pMsg += pExpr->param[j].nLen + 1; + pMsg += pExpr->param[j].nLen; } else { pSqlFuncExpr->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64Key); } @@ -1848,7 +1846,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->msgType = TSDB_MSG_TYPE_CM_HEARTBEAT; assert(msgLen + minMsgSize() <= size); - return msgLen; + return TSDB_CODE_SUCCESS; } int tscProcessTableMetaRsp(SSqlObj *pSql) { diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 4320e8f81b..bcc231c3f2 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -458,20 +458,21 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF } } -static char *getArithemicInputSrc(void *param, char *name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *)param; - SArithExprInfo * pExpr = pSupport->pArithExpr; +static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { +// SArithmeticSupport *pSupport = (SArithmeticSupport *)param; +// SArithExprInfo * pExpr = pSupport->pArithExpr; - int32_t index = -1; - for (int32_t i = 0; i < pExpr->binExprInfo.numOfCols; ++i) { - if (strcmp(name, pExpr->binExprInfo.pReqColumns[i].name) == 0) { - index = i; - break; - } - } - - assert(index >= 0 && index < pExpr->binExprInfo.numOfCols); - return pSupport->data[index] + pSupport->offset * pSupport->elemSize[index]; +// int32_t index = -1; +// for (int32_t i = 0; i < pExpr->numOfCols; ++i) { +// if (strcmp(name, pExpr->colList[i].name) == 0) { +// index = i; +// break; +// } +// } +// +// assert(index >= 0 && index < pExpr->numOfCols); +// return pSupport->data[index] + pSupport->offset * pSupport->elemSize[index]; +return 0; } static void **doSetResultRowData(SSqlObj *pSql) { @@ -521,21 +522,21 @@ static void **doSetResultRowData(SSqlObj *pSql) { sas->offset = 0; sas->pArithExpr = pInfo->pArithExprInfo; - sas->numOfCols = sas->pArithExpr->binExprInfo.numOfCols; +// sas->numOfCols = sas->pArithExpr->numOfCols; if (pRes->buffer[i] == NULL) { pRes->buffer[i] = malloc(tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i)->bytes); } for(int32_t k = 0; k < sas->numOfCols; ++k) { - int32_t columnIndex = sas->pArithExpr->binExprInfo.pReqColumns[k].colIndex; - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); - - sas->elemSize[k] = pExpr->resBytes; - sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; +// int32_t columnIndex = sas->pArithExpr->colList[k].colIndex; +// SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex); +// +// sas->elemSize[k] = pExpr->resBytes; +// sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes; } - tSQLBinaryExprCalcTraverse(sas->pArithExpr->binExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); + tExprTreeCalcTraverse(sas->pArithExpr->pExpr, 1, pRes->buffer[i], sas, TSDB_ORDER_ASC, getArithemicInputSrc); pRes->tsrow[i] = pRes->buffer[i]; free(sas); //todo optimization @@ -634,8 +635,6 @@ static UNUSED_FUNC void **tscBuildResFromSubqueries(SSqlObj *pSql) { } if (success) { // current row of final output has been built, return to app - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < numOfExprs; ++i) { int32_t tableIndex = pRes->pColumnIndex[i].tableIndex; int32_t columnIndex = pRes->pColumnIndex[i].columnIndex; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index b6d13554bb..856c28eaff 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -214,8 +214,8 @@ bool needSecondaryQuery(SQueryInfo* pQueryInfo) { size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pBase = taosArrayGet(pQueryInfo->colList, i); - if (pBase->colIndex.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + SColumn* base = taosArrayGet(pQueryInfo->colList, i); + if (base->colIndex.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) { return true; } } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index f1b6065f1c..75ab9fc720 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -17,8 +17,6 @@ #include "taosmsg.h" #include "tcache.h" #include "trpc.h" -#include "taosdef.h" -#include "tsocket.h" #include "tsystem.h" #include "ttime.h" #include "ttimer.h" @@ -34,7 +32,6 @@ // global, not configurable void * pVnodeConn; void * tscCacheHandle; -int slaveIndex; void * tscTmr; void * tscQhandle; void * tscCheckDiskUsageTmr; @@ -46,7 +43,7 @@ static pthread_once_t tscinit = PTHREAD_ONCE_INIT; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); -void tscCheckDiskUsage(void *para, void *unused) { +void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { taosGetDisk(); taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); } @@ -156,7 +153,6 @@ void taos_init_imp() { } tscInitMsgsFp(); - slaveIndex = rand(); int queueSize = tsMaxVnodeConnections + tsMaxMeterConnections + tsMaxMgmtConnections + tsMaxMgmtConnections; if (tscEmbedded == 0) { @@ -379,7 +375,6 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { return 0; } - int taos_options(TSDB_OPTION option, const void *arg, ...) { static int32_t lock = 0; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index e9ff558b92..698fa5ffe1 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -939,8 +939,9 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, i); if (pInfo->pArithExprInfo != NULL) { - tExprTreeDestroy(&pInfo->pArithExprInfo->binExprInfo.pBinExpr, NULL); - tfree(pInfo->pArithExprInfo->binExprInfo.pReqColumns); + tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); + assert(0); +// tfree(pInfo->pArithExprInfo->colList); } } @@ -1678,7 +1679,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST } if (pTagCols == NULL) { - pTableMetaInfo->tagColList = taosArrayInit(4, sizeof(SColumnIndex)); + pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); } else { pTableMetaInfo->tagColList = taosArrayClone(pTagCols); } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index dd6dc52242..3d5f798b13 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -356,17 +356,9 @@ typedef struct { } SMDDropVnodeMsg; typedef struct SColIndex { - int16_t colId; - /* - * colIdx is the index of column in latest schema of table - * it is available in the client side. Also used to determine - * whether current table schema is up-to-date. - * - * colIdxInBuf is used to denote the index of column in pQuery->colList, - * this value is invalid in client side, as well as in cache block of vnode either. - */ - int16_t colIndex; - uint16_t flag; // denote if it is a tag or not + int16_t colId; // column id + int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag + uint16_t flag; // denote if it is a tag or a normal column char name[TSDB_COL_NAME_LEN]; } SColIndex; @@ -388,15 +380,9 @@ typedef struct SSqlFuncMsg { } arg[3]; } SSqlFuncMsg; -typedef struct SExprInfo { - struct tExprNode *pBinExpr; /* for binary expression */ - int32_t numOfCols; /* binary expression involves the readed number of columns*/ - SColIndex * pReqColumns; /* source column list */ -} SExprInfo; - typedef struct SArithExprInfo { - SSqlFuncMsg pBase; - SExprInfo binExprInfo; + SSqlFuncMsg base; + struct tExprNode* pExpr; int16_t bytes; int16_t type; int16_t interResBytes; @@ -794,12 +780,12 @@ typedef struct { typedef struct { int32_t numOfQueries; - SQueryDesc *qdesc; + SQueryDesc qdesc[]; } SQqueryList; typedef struct { int32_t numOfStreams; - SStreamDesc *sdesc; + SStreamDesc sdesc[]; } SStreamList; typedef struct { diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index f3484509f8..b7b553dae7 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -82,10 +82,11 @@ void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param); -void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, - char *(*cb)(void *, char *, int32_t)); +void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, + char *(*cb)(void *, const char*, int32_t)); -void tSQLBinaryExprTrv(tExprNode *pExprs, int32_t *val, int16_t *ids); +// todo refactor: remove it +void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res); uint8_t getBinaryExprOptr(SSQLToken *pToken); diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 54055e9d33..7ac0609a55 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -115,10 +115,10 @@ enum { typedef struct SArithmeticSupport { SArithExprInfo *pArithExpr; - int32_t elemSize[TSDB_MAX_COLUMNS]; int32_t numOfCols; + SColumnInfo* colList; int32_t offset; - char * data[TSDB_MAX_COLUMNS]; + char** data; } SArithmeticSupport; typedef struct SQLPreAggVal { diff --git a/src/query/inc/tvariant.h b/src/query/inc/tvariant.h index 7ed5aa8dbe..05c7358d5f 100644 --- a/src/query/inc/tvariant.h +++ b/src/query/inc/tvariant.h @@ -38,7 +38,7 @@ void tVariantCreate(tVariant *pVar, SSQLToken *token); void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type); -void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t type); +void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32_t type); void tVariantDestroy(tVariant *pV); diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 14f340b3d3..d436d96d88 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -609,17 +609,6 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty } } -/* - * qsort comparator - * sort the result to ensure meters with the same gid is grouped together - */ -//static int32_t compareByAddr(const void *pLeft, const void *pRight) { -// int64_t p1 = (int64_t) * ((tSkipListNode **)pLeft); -// int64_t p2 = (int64_t) * ((tSkipListNode **)pRight); -// -// DEFAULT_COMP(p1, p2); -//} - int32_t merge(SArray *pLeft, SArray *pRight, SArray *pFinalRes) { // assert(pFinalRes->pRes == 0); // @@ -881,8 +870,8 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S } } -void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, - char *(*getSourceDataBlock)(void *, char *, int32_t)) { +void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, + char *(*getSourceDataBlock)(void *, const char*, int32_t)) { if (pExprs == NULL) { return; } @@ -893,13 +882,13 @@ void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOut /* the left output has result from the left child syntax tree */ char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows); if (pLeft->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprCalcTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); + tExprTreeCalcTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); } /* the right output has result from the right child syntax tree */ char *pRightOutput = malloc(sizeof(int64_t) * numOfRows); if (pRight->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); + tExprTreeCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); } if (pLeft->nodeType == TSQL_NODE_EXPR) { @@ -961,7 +950,7 @@ void tSQLBinaryExprCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOut free(pRightOutput); } -void tSQLBinaryExprTrv(tExprNode *pExprs, int32_t *val, int16_t *ids) { +void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) { if (pExprs == NULL) { return; } @@ -971,17 +960,15 @@ void tSQLBinaryExprTrv(tExprNode *pExprs, int32_t *val, int16_t *ids) { // recursive traverse left child branch if (pLeft->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprTrv(pLeft, val, ids); + tSQLBinaryExprTrv(pLeft, res); } else if (pLeft->nodeType == TSQL_NODE_COL) { - ids[*val] = pLeft->pSchema->colId; - (*val) += 1; + taosArrayPush(res, &pLeft->pSchema->colId); } if (pRight->nodeType == TSQL_NODE_EXPR) { - tSQLBinaryExprTrv(pRight, val, ids); + tSQLBinaryExprTrv(pRight, res); } else if (pRight->nodeType == TSQL_NODE_COL) { - ids[*val] = pRight->pSchema->colId; - (*val) += 1; + taosArrayPush(res, &pRight->pSchema->colId); } } diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 1cb7918ed5..f4f5c48f9c 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -53,8 +53,8 @@ /* get the qinfo struct address from the query struct address */ #define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].bytes) -#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].pBase.colInfo.colIndex].type) + ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].bytes) +#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].type) typedef struct SPointInterpoSupporter { int32_t numOfCols; @@ -257,7 +257,7 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { int64_t maxOutput = 0; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[j].base.functionId; /* * ts, tag, tagprj function can not decide the output number of current query @@ -334,7 +334,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { int32_t numOfSelectivity = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functId = pQuery->pSelectExpr[i].base.functionId; if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { hasTags = true; continue; @@ -352,7 +352,7 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { return false; } -bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].pBase.functionId == TSDB_FUNC_TS_COMP; } +bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.functionId == TSDB_FUNC_TS_COMP; } static bool limitResults(SQInfo *pQInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -370,7 +370,7 @@ static bool limitResults(SQInfo *pQInfo) { static bool isTopBottomQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TS) { continue; } @@ -385,7 +385,7 @@ static bool isTopBottomQuery(SQuery *pQuery) { static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, int32_t index) { // for a tag column, no corresponding field info - SColIndex *pColIndexEx = &pQuery->pSelectExpr[index].pBase.colInfo; + SColIndex *pColIndexEx = &pQuery->pSelectExpr[index].base.colInfo; if (TSDB_COL_IS_TAG(pColIndexEx->flag)) { return NULL; } @@ -413,7 +413,7 @@ static SDataStatis *getStatisInfo(SQuery *pQuery, SDataStatis *pStatis, SDataBlo */ static bool hasNullValue(SQuery *pQuery, int32_t col, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, SDataStatis **pColStatis) { - SColIndex *pColIndex = &pQuery->pSelectExpr[col].pBase.colInfo; + SColIndex *pColIndex = &pQuery->pSelectExpr[col].base.colInfo; if (TSDB_COL_IS_TAG(pColIndex->flag)) { return false; } @@ -717,7 +717,7 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; pCtx[k].nStartQueryTimestamp = pWin->skey; pCtx[k].size = forwardStep; @@ -743,7 +743,7 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].nStartQueryTimestamp = pWin->skey; - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunctionF(&pCtx[k], offset); } @@ -817,13 +817,12 @@ static TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) { char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock) { - SQuery * pQuery = pRuntimeEnv->pQuery; + char *dataBlock = NULL; + SQuery *pQuery = pRuntimeEnv->pQuery; + SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - char *dataBlock = NULL; - - int32_t functionId = pQuery->pSelectExpr[col].pBase.functionId; - + int32_t functionId = pQuery->pSelectExpr[col].base.functionId; if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pSelectExpr[col]; @@ -833,19 +832,33 @@ char *getDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int3 } else { pCtx->startOffset = pQuery->pos - (size - 1); } - + + sas->offset = 0; + sas->colList = pQuery->colList; + sas->numOfCols = pQuery->numOfCols; + sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); + + // here the pQuery->colList and sas->colList are identical for (int32_t i = 0; i < pQuery->numOfCols; ++i) { SColumnInfo *pColMsg = &pQuery->colList[i]; - assert(0); -// char * pData = doGetDataBlocks(pQuery, pRuntimeEnv->colDataBuffer, pQuery->colList[i].colIdxInBuf); - sas->elemSize[i] = pColMsg->bytes; - // sas->data[i] = pData + pCtx->startOffset * sas->elemSize[i]; // start from the offset + + int32_t numOfCols = taosArrayGetSize(pDataBlock); + + dataBlock = NULL; + for (int32_t k = 0; k < numOfCols; ++k) { //todo refactor + SColumnInfoData *p = taosArrayGet(pDataBlock, k); + if (pColMsg->colId == p->info.colId) { + dataBlock = p->pData; + break; + } + } + + assert(dataBlock != NULL); + sas->data[i] = dataBlock + pCtx->startOffset * pQuery->colList[i].bytes; // start from the offset } - sas->numOfCols = pQuery->numOfCols; - sas->offset = 0; } else { // other type of query function - SColIndex *pCol = &pQuery->pSelectExpr[col].pBase.colInfo; + SColIndex *pCol = &pQuery->pSelectExpr[col].base.colInfo; if (TSDB_COL_IS_TAG(pCol->flag) || pDataBlock == NULL) { dataBlock = NULL; } else { @@ -894,9 +907,9 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; SDataStatis *tpField = NULL; @@ -953,13 +966,21 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * * tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY */ for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunction(&pCtx[k]); } } } + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + if (pQuery->pSelectExpr[i].base.functionId != TSDB_FUNC_ARITHM) { + continue; + } + + tfree(sasArray[i].data); + } + tfree(sasArray); } @@ -1103,7 +1124,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; SDataStatis *pColStatis = NULL; @@ -1212,7 +1233,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS offset -= pCtx[0].startOffset; for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pSelectExpr[k].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[k].base.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { aAggs[functionId].xFunctionF(&pCtx[k], offset); } @@ -1229,6 +1250,16 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } pQuery->lastKey = lastKey + step; + + // todo refactor: extract method + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + if (pQuery->pSelectExpr[i].base.functionId != TSDB_FUNC_ARITHM) { + continue; + } + + tfree(sasArray[i].data); + } + free(sasArray); } @@ -1343,7 +1374,7 @@ static void setCtxTagColumnInfo(SQuery *pQuery, SQLFunctionCtx *pCtx) { SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; @@ -1383,7 +1414,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pRuntimeEnv->offset[0] = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].pBase; + SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; SColIndex* pIndex = &pSqlFuncMsg->colInfo; @@ -1425,7 +1456,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int32_t functionId = pCtx->functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - int32_t f = pQuery->pSelectExpr[0].pBase.functionId; + int32_t f = pQuery->pSelectExpr[0].base.functionId; assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); pCtx->param[2].i64Key = order; @@ -1533,7 +1564,7 @@ bool isFixedOutputQuery(SQuery *pQuery) { } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].base; // ignore the ts_comp function if (i == 0 && pExprMsg->functionId == TSDB_FUNC_PRJ && pExprMsg->numOfParams == 1 && @@ -1555,7 +1586,7 @@ bool isFixedOutputQuery(SQuery *pQuery) { bool isPointInterpoQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionID = pQuery->pSelectExpr[i].base.functionId; if (functionID == TSDB_FUNC_INTERP || functionID == TSDB_FUNC_LAST_ROW) { return true; } @@ -1567,7 +1598,7 @@ bool isPointInterpoQuery(SQuery *pQuery) { // TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION bool isSumAvgRateQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TS) { continue; } @@ -1583,7 +1614,7 @@ bool isSumAvgRateQuery(SQuery *pQuery) { bool isFirstLastRowQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionID = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionID = pQuery->pSelectExpr[i].base.functionId; if (functionID == TSDB_FUNC_LAST_ROW) { return true; } @@ -1599,7 +1630,7 @@ bool notHasQueryTimeRange(SQuery *pQuery) { static bool needReverseScan(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) { continue; } @@ -1772,7 +1803,7 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { } else { bool hasMultioutput = false; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].pBase; + SSqlFuncMsg *pExprMsg = &pQuery->pSelectExpr[i].base; if (pExprMsg->functionId == TSDB_FUNC_TS || pExprMsg->functionId == TSDB_FUNC_TS_DUMMY) { continue; } @@ -1805,7 +1836,7 @@ bool vnodeParametersSafetyCheck(SQuery *pQuery) { // the scan order is not matter static bool onlyOneQueryType(SQuery *pQuery, int32_t functId, int32_t functIdDst) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAG_DUMMY) { @@ -2008,7 +2039,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI SInterpInfoDetail *pInterpDetail = pInterpInfo->pInterpDetail; // for primary timestamp column, set the flag - if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pQuery->pSelectExpr[i].base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { pInterpDetail->primaryCol = 1; } @@ -2031,11 +2062,11 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; // tag column does not need the interp environment - if (pQuery->pSelectExpr[i].pBase.functionId == TSDB_FUNC_TAG) { + if (pQuery->pSelectExpr[i].base.functionId == TSDB_FUNC_TAG) { continue; } - int32_t colInBuf = 0; // pQuery->pSelectExpr[i].pBase.colInfo.colIdxInBuf; + int32_t colInBuf = 0; // pQuery->pSelectExpr[i].base.colInfo.colIdxInBuf; SInterpInfo *pInterpInfo = (SInterpInfo *)pRuntimeEnv->pCtx[i].aOutputBuf; pInterpInfo->pInterpDetail = calloc(1, sizeof(SInterpInfoDetail)); @@ -2046,7 +2077,7 @@ void pointInterpSupporterSetData(SQInfo *pQInfo, SPointInterpoSupporter *pPointI assert(0); // for primary timestamp column, set the flag - if (pQuery->pSelectExpr[i].pBase.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pQuery->pSelectExpr[i].base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { pInterpDetail->primaryCol = 1; } else { doSetInterpVal(pCtx, prevKey, type, 1, pPointInterpSupport->pPrevPoint[colInBuf]); @@ -2145,7 +2176,7 @@ static int32_t getRowParamForMultiRowsOutput(SQuery *pQuery, bool isSTableQuery) int32_t rowparam = 1; if (isTopBottomQuery(pQuery) && (!isSTableQuery)) { - rowparam = pQuery->pSelectExpr[1].pBase.arg->argValue.i64; + rowparam = pQuery->pSelectExpr[1].base.arg->argValue.i64; } return rowparam; @@ -2211,7 +2242,7 @@ UNUSED_FUNC void setTimestampRange(SQueryRuntimeEnv *pRuntimeEnv, int64_t stime, SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_SPREAD) { pRuntimeEnv->pCtx[i].param[1].dKey = stime; @@ -2270,7 +2301,7 @@ static bool needToLoadDataBlock(SQuery *pQuery, SDataStatis *pDataStatis, SQLFun // todo disable this opt code block temporarily // for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - // int32_t functId = pQuery->pSelectExpr[i].pBase.functionId; + // int32_t functId = pQuery->pSelectExpr[i].base.functionId; // if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { // return top_bot_datablock_filter(&pCtx[i], functId, (char *)&pField[i].min, (char *)&pField[i].max); // } @@ -2298,8 +2329,8 @@ SArray *loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBl r = BLK_DATA_ALL_NEEDED; } else { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; - int32_t colId = pQuery->pSelectExpr[i].pBase.colInfo.colId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; + int32_t colId = pQuery->pSelectExpr[i].base.colInfo.colId; r |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pQuery->window.skey, pQuery->window.ekey, colId); } @@ -2533,14 +2564,14 @@ static void doSetTagValueInParam(void *tsdb, STableId id, int32_t tagColId, tVar void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, STableId id, void *tsdb) { SQuery *pQuery = pRuntimeEnv->pQuery; - SSqlFuncMsg *pFuncMsg = &pQuery->pSelectExpr[0].pBase; + SSqlFuncMsg *pFuncMsg = &pQuery->pSelectExpr[0].base; if (pQuery->numOfOutput == 1 && pFuncMsg->functionId == TSDB_FUNC_TS_COMP) { assert(pFuncMsg->numOfParams == 1); doSetTagValueInParam(tsdb, id, pFuncMsg->arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag); } else { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { - SColIndex *pCol = &pQuery->pSelectExpr[idx].pBase.colInfo; + SColIndex *pCol = &pQuery->pSelectExpr[idx].base.colInfo; // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pCol->flag)) { @@ -2566,7 +2597,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (!mergeFlag) { pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; pCtx[i].currentStage = FIRST_STAGE_MERGE; @@ -2590,7 +2621,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TAG_DUMMY) { continue; } @@ -2680,15 +2711,15 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf for (int32_t i = 0; i < numOfCols; ++i) { switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { - int32_t colIndex = pQuery->pSelectExpr[i].pBase.colInfo.colIndex; + int32_t colIndex = pQuery->pSelectExpr[i].base.colInfo.colIndex; int32_t type = 0; - if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].pBase.colInfo.flag)) { + if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].base.colInfo.flag)) { type = pQuery->pSelectExpr[i].type; } else { type = pMeterObj->schema[colIndex].type; } - printBinaryData(pQuery->pSelectExpr[i].pBase.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j, + printBinaryData(pQuery->pSelectExpr[i].base.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j, type); break; } @@ -2843,7 +2874,7 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pW int64_t maxOutput = 0; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[j].base.functionId; /* * ts, tag, tagprj function can not decide the output number of current query @@ -3059,7 +3090,7 @@ static void doDisableFunctsForSupplementaryScan(SQuery *pQuery, SWindowResInfo * // open/close the specified query for each group result for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; + int32_t functId = pQuery->pSelectExpr[j].base.functionId; if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { @@ -3081,7 +3112,7 @@ void disableFuncInReverseScan(SQueryRuntimeEnv *pRuntimeEnv) { doDisableFunctsForSupplementaryScan(pQuery, pWindowResInfo, order); } else { // for simple result of table query, for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functId = pQuery->pSelectExpr[j].pBase.functionId; + int32_t functId = pQuery->pSelectExpr[j].base.functionId; SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; @@ -3151,7 +3182,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; // set the timestamp output buffer for top/bottom/diff query - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } @@ -3167,7 +3198,7 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { // reset the execution contexts for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[j].base.functionId; assert(functionId != TSDB_FUNC_DIFF); // set next output position @@ -3194,7 +3225,7 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pSelectExpr[j].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[j].base.functionId; pRuntimeEnv->pCtx[j].currentStage = 0; aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); @@ -3220,7 +3251,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->rec.rows -= numOfSkip; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; memmove(pQuery->sdata[i]->data, pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->rec.rows * bytes); @@ -3262,7 +3293,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { setWindowResOutputBuf(pRuntimeEnv, pResult); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; + int16_t functId = pQuery->pSelectExpr[j].base.functionId; if (functId == TSDB_FUNC_TS) { continue; } @@ -3275,7 +3306,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } } else { for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int16_t functId = pQuery->pSelectExpr[j].pBase.functionId; + int16_t functId = pQuery->pSelectExpr[j].base.functionId; if (functId == TSDB_FUNC_TS) { continue; } @@ -3443,7 +3474,7 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { setWindowResOutputBuf(pRuntimeEnv, buf); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); } /* @@ -3455,14 +3486,14 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { } else { for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - aAggs[pQuery->pSelectExpr[j].pBase.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); } } } static bool hasMainOutput(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TAGPRJ) { return true; @@ -3568,7 +3599,7 @@ static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } @@ -3666,7 +3697,7 @@ void setIntervalQueryRange(STableQueryInfo *pTableQueryInfo, SQInfo *pQInfo, TSK bool requireTimestamp(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; i++) { - int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId; + int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_NEED_TS) != 0) { return true; } @@ -3883,7 +3914,7 @@ static UNUSED_FUNC int32_t resultInterpolate(SQInfo *pQInfo, tFilePage **data, t for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { srcData[i] = pDataSrc[i]->data; - functions[i] = pQuery->pSelectExpr[i].pBase.functionId; + functions[i] = pQuery->pSelectExpr[i].base.functionId; } assert(0); @@ -5337,9 +5368,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pDestFilterInfo->filterstr) { pDestFilterInfo->len = htobe64(pFilterInfo->len); - pDestFilterInfo->pz = (int64_t)calloc(1, pDestFilterInfo->len + 1); - memcpy((void *)pDestFilterInfo->pz, pMsg, pDestFilterInfo->len + 1); - pMsg += (pDestFilterInfo->len + 1); + pDestFilterInfo->pz = (int64_t) calloc(1, pDestFilterInfo->len); + memcpy((void *)pDestFilterInfo->pz, pMsg, pDestFilterInfo->len); + pMsg += (pDestFilterInfo->len); } else { pDestFilterInfo->lowerBndi = htobe64(pFilterInfo->lowerBndi); pDestFilterInfo->upperBndi = htobe64(pFilterInfo->upperBndi); @@ -5372,7 +5403,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pExprMsg->arg[j].argType == TSDB_DATA_TYPE_BINARY) { pExprMsg->arg[j].argValue.pz = pMsg; - pMsg += pExprMsg->arg[j].argBytes + 1; // one more for the string terminated char. + pMsg += pExprMsg->arg[j].argBytes; // one more for the string terminated char. } else { pExprMsg->arg[j].argValue.i64 = htobe64(pExprMsg->arg[j].argValue.i64); } @@ -5435,28 +5466,29 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; } - - // the tag query condition expression string is located at the end of query msg - if (pQueryMsg->tagCondLen > 0) { - *tagCond = calloc(1, pQueryMsg->tagCondLen); - memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); - pMsg += pQueryMsg->tagCondLen; - } if (pQueryMsg->numOfTags > 0) { (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { SColumnInfo* pTagCol = (SColumnInfo*) pMsg; - + pTagCol->colId = htons(pTagCol->colId); pTagCol->bytes = htons(pTagCol->bytes); pTagCol->type = htons(pTagCol->type); pTagCol->numOfFilters = 0; (*tagCols)[i] = *pTagCol; + pMsg += sizeof(SColumnInfo); } } + // the tag query condition expression string is located at the end of query msg + if (pQueryMsg->tagCondLen > 0) { + *tagCond = calloc(1, pQueryMsg->tagCondLen); + memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); + pMsg += pQueryMsg->tagCondLen; + } + qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, ts order:%d, " "outputCols:%d, numOfCols:%d, interval:%d" PRId64 ", fillType:%d, comptsLen:%d, limit:%" PRId64 @@ -5468,54 +5500,38 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, return 0; } -static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pExpr, SQueryTableMsg *pQueryMsg) { -// SExprInfo *pBinaryExprInfo = &pExpr->binExprInfo; -// SColumnInfo * pColMsg = pQueryMsg->colList; -#if 0 - tExprNode* pBinExpr = NULL; - SSchema* pSchema = toSchema(pQueryMsg, pColMsg, pQueryMsg->numOfCols); +//static int32_t id_compar(const void* p1, const void* p2) { +// return GET_INT16_VAL(p1) - GET_INT16_VAL(p2); +//} + +static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) { + tExprNode* pExprNode = NULL; - qTrace("qmsg:%p create binary expr from string:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz); - tSQLBinaryExprFromString(&pBinExpr, pSchema, pQueryMsg->numOfCols, pExpr->pBase.arg[0].argValue.pz, - pExpr->pBase.arg[0].argBytes); + qTrace("qmsg:%p create arithmetic expr from binary string", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz); - if (pBinExpr == NULL) { - qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExpr->pBase.arg[0].argValue.pz); + int32_t ret = exprTreeFromBinary(pArithExprInfo->base.arg[0].argValue.pz, pArithExprInfo->base.arg[0].argBytes, &pExprNode); + if (pExprNode == NULL || ret != TSDB_CODE_SUCCESS) { + qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz); return TSDB_CODE_APP_ERROR; } - pBinaryExprInfo->pBinExpr = pBinExpr; + pArithExprInfo->pExpr = pExprNode; - int32_t num = 0; - int16_t ids[TSDB_MAX_COLUMNS] = {0}; +// SArray* res = taosArrayInit(4, sizeof(int16_t)); +// tSQLBinaryExprTrv(pExprNode, res); - tSQLBinaryExprTrv(pBinExpr, &num, ids); - qsort(ids, num, sizeof(int16_t), id_compar); - - int32_t i = 0, j = 0; - - while (i < num && j < num) { - if (ids[i] == ids[j]) { - j++; - } else { - ids[++i] = ids[j++]; - } - } - assert(i <= num); +// size_t num = taosArrayGetSize(res); +// qsort(res->pData, num, sizeof(int16_t), id_compar); // there may be duplicated referenced columns. - num = i + 1; - pBinaryExprInfo->pReqColumns = malloc(sizeof(SColIndex) * num); +// pArithExprInfo->colList = calloc(pQueryMsg->numOfCols, sizeof(SColIndex)); - for (int32_t k = 0; k < num; ++k) { - SColIndex* pColIndex = &pBinaryExprInfo->pReqColumns[k]; - pColIndex->colId = ids[k]; - } +// for (int32_t k = 0; k < pQueryMsg->numOfCols; ++k) { +// SColIndex* pColIndex = &pArithExprInfo->colList[k]; +// pColIndex->colId = pQueryMsg->colList[k].colId; +// } - pBinaryExprInfo->numOfCols = num; - free(pSchema); -#endif - +// pArithExprInfo->numOfCols = pQueryMsg->numOfCols; return TSDB_CODE_SUCCESS; } @@ -5533,14 +5549,14 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExp int16_t tagLen = 0; for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - pExprs[i].pBase = *pExprMsg[i]; + pExprs[i].base = *pExprMsg[i]; pExprs[i].bytes = 0; int16_t type = 0; int16_t bytes = 0; // parse the arithmetic expression - if (pExprs[i].pBase.functionId == TSDB_FUNC_ARITHM) { + if (pExprs[i].base.functionId == TSDB_FUNC_ARITHM) { code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); if (code != TSDB_CODE_SUCCESS) { @@ -5550,26 +5566,26 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExp type = TSDB_DATA_TYPE_DOUBLE; bytes = tDataTypeDesc[type].nSize; - } else if (pExprs[i].pBase.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { // parse the normal column + } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { // parse the normal column type = TSDB_DATA_TYPE_BINARY; bytes = TSDB_TABLE_NAME_LEN; } else{ - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase, pTagCols); + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols); - SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].pBase.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j]; + SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j]; type = pCol->type; bytes = pCol->bytes; } - int32_t param = pExprs[i].pBase.arg[0].argValue.i64; - if (getResultDataInfo(type, bytes, pExprs[i].pBase.functionId, param, &pExprs[i].type, &pExprs[i].bytes, + int32_t param = pExprs[i].base.arg[0].argValue.i64; + if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { tfree(pExprs); return TSDB_CODE_INVALID_QUERY_MSG; } - if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].pBase.functionId == TSDB_FUNC_TS_DUMMY) { + if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExprs[i].bytes; } assert(isValidDataType(pExprs[i].type, pExprs[i].bytes)); @@ -5579,17 +5595,17 @@ static int32_t createSqlFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SArithExp // TODO refactor for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - pExprs[i].pBase = *pExprMsg[i]; - int16_t functId = pExprs[i].pBase.functionId; + pExprs[i].base = *pExprMsg[i]; + int16_t functId = pExprs[i].base.functionId; if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].pBase, pTagCols); + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols); SColumnInfo *pCol = &pQueryMsg->colList[j]; int32_t ret = - getResultDataInfo(pCol->type, pCol->bytes, functId, pExprs[i].pBase.arg[0].argValue.i64, + getResultDataInfo(pCol->type, pCol->bytes, functId, pExprs[i].base.arg[0].argValue.i64, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interResBytes, tagLen, isSuperTable); assert(ret == TSDB_CODE_SUCCESS); } @@ -5714,7 +5730,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { assert(pQuery->pSelectExpr != NULL && pQuery != NULL); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - SSqlFuncMsg *pSqlExprMsg = &pQuery->pSelectExpr[k].pBase; + SSqlFuncMsg *pSqlExprMsg = &pQuery->pSelectExpr[k].base; if (pSqlExprMsg->functionId == TSDB_FUNC_ARITHM || pSqlExprMsg->colInfo.flag == TSDB_COL_TAG) { continue; } @@ -5765,6 +5781,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->interpoType = pQueryMsg->interpoType; pQuery->numOfTags = pQueryMsg->numOfTags; + // todo do not allocate ?? pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); if (pQuery->colList == NULL) { goto _cleanup; @@ -5942,12 +5959,12 @@ static void freeQInfo(SQInfo *pQInfo) { if (pQuery->pSelectExpr != NULL) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].binExprInfo; +// SExprInfo *pBinExprInfo = &pQuery->pSelectExpr[i].arithExprInfo; - if (pBinExprInfo->numOfCols > 0) { - tfree(pBinExprInfo->pReqColumns); - tExprTreeDestroy(&pBinExprInfo->pBinExpr, NULL); - } +// if (pBinExprInfo->numOfCols > 0) { +// tfree(pBinExprInfo->pReqColumns); +// tExprTreeDestroy(&pBinExprInfo->pBinExpr, NULL); +// } } tfree(pQuery->pSelectExpr); diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index f0addb733b..31f04d337d 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -72,7 +72,7 @@ void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t t * @param len * @param type */ -void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { +void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { @@ -109,10 +109,10 @@ void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t t break; } - case TSDB_DATA_TYPE_BINARY: { - pVar->pz = strndup(pz, len); - pVar->nLen = strdequote(pVar->pz); - + case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method + pVar->pz = calloc(len, sizeof(char)); + memcpy(pVar->pz, pz, len); + pVar->nLen = len; break; } From e82f42a52bf7acaafd06abfde7596072524d9fb9 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sat, 25 Apr 2020 23:05:15 +0800 Subject: [PATCH 49/63] [td-197] refactor codes, and remove unused attributes in querymsg --- src/client/src/tscServer.c | 23 ----------------------- src/inc/taosmsg.h | 4 ++-- src/query/src/queryExecutor.c | 33 +-------------------------------- 3 files changed, 3 insertions(+), 57 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index af623596a4..48935480ad 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -758,16 +758,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } - bool hasArithmeticFunction = false; - SSqlFuncMsg *pSqlFuncExpr = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->functionId == TSDB_FUNC_ARITHM) { - hasArithmeticFunction = true; - } - if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); @@ -797,23 +791,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSqlFuncExpr = (SSqlFuncMsg *)pMsg; } - int32_t len = 0; - if (hasArithmeticFunction) { - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pColBase = taosArrayGetP(pQueryInfo->colList, i); - - char * name = pSchema[pColBase[i].colIndex.columnIndex].name; - int32_t lenx = strlen(name); - memcpy(pMsg, name, lenx); - *(pMsg + lenx) = ','; - - len += (lenx + 1); // one for comma - pMsg += (lenx + 1); - } - } - - pQueryMsg->colNameLen = htonl(len); - // serialize the table info (sid, uid, tags) pMsg = doSerializeTableInfo(pSql, htons(pQueryMsg->head.vgId), pMsg); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 3d5f798b13..b89f8b871f 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -459,8 +459,8 @@ typedef struct { int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list - int32_t colNameLen; - int64_t colNameList; +// int32_t colNameLen; +// int64_t colNameList; int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed int32_t tsLen; // total length of ts comp block int32_t tsNumOfBlocks; // ts comp block numbers diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index f4f5c48f9c..b19e024dd9 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5381,8 +5381,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } } - bool hasArithmeticFunction = false; - *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; @@ -5409,9 +5407,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } } - if (pExprMsg->functionId == TSDB_FUNC_ARITHM) { - hasArithmeticFunction = true; - } else if (pExprMsg->functionId == TSDB_FUNC_TAG || pExprMsg->functionId == TSDB_FUNC_TAGPRJ || + if (pExprMsg->functionId == TSDB_FUNC_TAG || pExprMsg->functionId == TSDB_FUNC_TAGPRJ || pExprMsg->functionId == TSDB_FUNC_TAG_DUMMY) { if (pExprMsg->colInfo.flag != TSDB_COL_TAG) { // ignore the column index check for arithmetic expression. return TSDB_CODE_INVALID_QUERY_MSG; @@ -5425,13 +5421,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pExprMsg = (SSqlFuncMsg *)pMsg; } - pQueryMsg->colNameLen = htonl(pQueryMsg->colNameLen); - if (hasArithmeticFunction) { // column name array - assert(pQueryMsg->colNameLen > 0); - pQueryMsg->colNameList = (int64_t)pMsg; - pMsg += pQueryMsg->colNameLen; - } - pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns @@ -5500,10 +5489,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, return 0; } -//static int32_t id_compar(const void* p1, const void* p2) { -// return GET_INT16_VAL(p1) - GET_INT16_VAL(p2); -//} - static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) { tExprNode* pExprNode = NULL; @@ -5516,22 +5501,6 @@ static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pArithExprInfo, SQuery } pArithExprInfo->pExpr = pExprNode; - -// SArray* res = taosArrayInit(4, sizeof(int16_t)); -// tSQLBinaryExprTrv(pExprNode, res); - -// size_t num = taosArrayGetSize(res); -// qsort(res->pData, num, sizeof(int16_t), id_compar); - - // there may be duplicated referenced columns. -// pArithExprInfo->colList = calloc(pQueryMsg->numOfCols, sizeof(SColIndex)); - -// for (int32_t k = 0; k < pQueryMsg->numOfCols; ++k) { -// SColIndex* pColIndex = &pArithExprInfo->colList[k]; -// pColIndex->colId = pQueryMsg->colList[k].colId; -// } - -// pArithExprInfo->numOfCols = pQueryMsg->numOfCols; return TSDB_CODE_SUCCESS; } From ad519de25f20d3ba4a6a0c77eb7ebd50832f2eb4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 25 Apr 2020 23:14:05 +0800 Subject: [PATCH 50/63] refine python testcases. --- tests/pytest/insert/basic.py | 1 - tests/pytest/insert/float.py | 3 --- tests/pytest/insert/int.py | 1 - tests/pytest/util/cases.py | 2 +- 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/pytest/insert/basic.py b/tests/pytest/insert/basic.py index e8698e9d05..35f830d951 100644 --- a/tests/pytest/insert/basic.py +++ b/tests/pytest/insert/basic.py @@ -12,7 +12,6 @@ # -*- coding: utf-8 -*- import sys -import taos from util.log import * from util.cases import * from util.sql import * diff --git a/tests/pytest/insert/float.py b/tests/pytest/insert/float.py index 7c4474f7ad..30d7e223f1 100644 --- a/tests/pytest/insert/float.py +++ b/tests/pytest/insert/float.py @@ -13,9 +13,6 @@ import sys import datetime - -import taos - from util.log import * from util.cases import * from util.sql import * diff --git a/tests/pytest/insert/int.py b/tests/pytest/insert/int.py index 8d38337576..d007cc56ea 100644 --- a/tests/pytest/insert/int.py +++ b/tests/pytest/insert/int.py @@ -12,7 +12,6 @@ # -*- coding: utf-8 -*- import sys -import taos import datetime from util.log import * diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 1771c23620..9a4f0fc98a 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -71,7 +71,7 @@ class TDCases: case.run() except Exception as e: tdLog.notice(repr(e)) - tdLog.notice("%s failed: %s" % (__file__, fileName)) + tdLog.exit("%s failed: %s" % (__file__, fileName)) case.stop() runNum += 1 continue From 60dc8cba6b3864c849880b12a16d9eccde9f1fc9 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 26 Apr 2020 09:27:16 +0800 Subject: [PATCH 51/63] [TD-90] fix bug in alter table --- src/client/src/tscServer.c | 4 +- src/inc/taoserror.h | 7 ++ src/inc/taosmsg.h | 24 +++---- src/mnode/src/mgmtTable.c | 129 ++++++++++++++++++++----------------- 4 files changed, 92 insertions(+), 72 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 21f6f9d390..c048714712 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1348,11 +1348,11 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name); pAlterTableMsg->type = htons(pAlterInfo->type); - pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); + pAlterTableMsg->numOfCols = tscNumOfFields(pQueryInfo); memcpy(pAlterTableMsg->tagVal, pAlterInfo->tagData.data, TSDB_MAX_TAGS_LEN); SSchema *pSchema = pAlterTableMsg->schema; - for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { + for (int i = 0; i < pAlterTableMsg->numOfCols; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pSchema->type = pField->type; diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 5084199ac8..c9071f4cb0 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -97,6 +97,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TABLE, 0, 203, "invalid table n TAOS_DEFINE_ERROR(TSDB_CODE_NOT_SUPER_TABLE, 0, 204, "no super table") // operation only available for super table TAOS_DEFINE_ERROR(TSDB_CODE_NOT_ACTIVE_TABLE, 0, 205, "not active table") TAOS_DEFINE_ERROR(TSDB_CODE_TABLE_ID_MISMATCH, 0, 206, "table id mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_TAG_ALREAY_EXIST, 0, 207, "tag already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_TAG_NOT_EXIST, 0, 208, "tag not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_ALREAY_EXIST, 0, 209, "field already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_FIELD_NOT_EXIST, 0, 210, "field not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_COL_NAME_TOO_LONG, 0, 211, "column name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_TOO_MANY_TAGS, 0, 211, "too many tags") + // dnode & mnode TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DNODES, 0, 250, "no enough dnodes") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 16e147c5af..8e6e762017 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -238,7 +238,7 @@ typedef struct { typedef struct SSchema { uint8_t type; - char name[TSDB_COL_NAME_LEN]; + char name[TSDB_COL_NAME_LEN + 1]; int16_t colId; int16_t bytes; } SSchema; @@ -256,14 +256,14 @@ typedef struct { uint64_t uid; uint64_t superTableUid; uint64_t createdTime; - char tableId[TSDB_TABLE_ID_LEN]; - char superTableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; + char superTableId[TSDB_TABLE_ID_LEN + 1]; char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - char db[TSDB_DB_NAME_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; + char db[TSDB_DB_NAME_LEN + 1]; int8_t igExists; int16_t numOfTags; int16_t numOfColumns; @@ -274,13 +274,13 @@ typedef struct { } SCMCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; int8_t igNotExists; } SCMDropTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; - char db[TSDB_DB_NAME_LEN]; + char tableId[TSDB_TABLE_ID_LEN + 1]; + char db[TSDB_DB_NAME_LEN + 1]; int16_t type; /* operation type */ char tagVal[TSDB_MAX_BYTES_PER_ROW]; int8_t numOfCols; /* number of schema */ @@ -515,8 +515,8 @@ typedef struct { } SVnodeLoad; typedef struct { - char acct[TSDB_USER_LEN]; - char db[TSDB_DB_NAME_LEN]; + char acct[TSDB_USER_LEN + 1]; + char db[TSDB_DB_NAME_LEN + 1]; uint32_t vgId; int32_t maxSessions; int32_t cacheBlockSize; @@ -713,8 +713,8 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; - char tableId[TSDB_TABLE_ID_LEN]; // table id - char stableId[TSDB_TABLE_ID_LEN]; // stable name if it is created according to super table + char tableId[TSDB_TABLE_ID_LEN + 1]; // table id + char stableId[TSDB_TABLE_ID_LEN + 1]; // stable name if it is created according to super table uint8_t numOfTags; uint8_t precision; uint8_t tableType; diff --git a/src/mnode/src/mgmtTable.c b/src/mnode/src/mgmtTable.c index 1813be6522..57e905361c 100644 --- a/src/mnode/src/mgmtTable.c +++ b/src/mnode/src/mgmtTable.c @@ -78,6 +78,8 @@ static void mgmtGetChildTableMeta(SQueuedMsg *pMsg); static void mgmtProcessAlterTableMsg(SQueuedMsg *queueMsg); static void mgmtProcessAlterTableRsp(SRpcMsg *rpcMsg); +static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName); + static void mgmtDestroyChildTable(SChildTableObj *pTable) { tfree(pTable->schema); tfree(pTable->sql); @@ -757,11 +759,11 @@ static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) { SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; if (pStable->numOfTables != 0) { mgmtDropAllChildTablesInStable(pStable); - for (int32_t vg = 0; vg < pTable->vgLen; ++vg) { - int32_t vgId = pTable->vgList[vg]; + for (int32_t vg = 0; vg < pStable->vgLen; ++vg) { + int32_t vgId = pStable->vgList[vg]; if (vgId == 0) break; - SMDDropSTableMsg *pDrop = rpcMalloc(sizeof(SMDDropSTableMsg)); + SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg)); pDrop->vgId = htonl(vgId); pDrop->uid = htobe64(pStable->uid); mgmtExtractTableName(pStable->info.tableId, pDrop->tableId); @@ -801,31 +803,33 @@ static int32_t mgmtFindSuperTableTagIndex(SSuperTableObj *pStable, const char *t static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], int32_t ntags) { if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { - return TSDB_CODE_APP_ERROR; + mError("stable:%s, add tag, too many tags", pStable->info.tableId); + return TSDB_CODE_TOO_MANY_TAGS; } - // check if schemas have the same name - for (int32_t i = 1; i < ntags; i++) { - for (int32_t j = 0; j < i; j++) { - if (strcasecmp(schema[i].name, schema[j].name) == 0) { - return TSDB_CODE_APP_ERROR; - } + for (int32_t i = 0; i < ntags; i++) { + if (mgmtFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { + mError("stable:%s, add tag, column:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_TAG_ALREAY_EXIST; + } + + if (mgmtFindSuperTableTagIndex(pStable, schema[i].name) > 0) { + mError("stable:%s, add tag, tag:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_FIELD_ALREAY_EXIST; } } int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize + sizeof(SSchema) * ntags); - memmove(pStable->schema + sizeof(SSchema) * (pStable->numOfColumns + ntags), - pStable->schema + sizeof(SSchema) * pStable->numOfColumns, sizeof(SSchema) * pStable->numOfTags); - memcpy(pStable->schema + sizeof(SSchema) * pStable->numOfColumns, schema, sizeof(SSchema) * ntags); + memcpy(pStable->schema + pStable->numOfColumns + pStable->numOfTags, schema, sizeof(SSchema) * ntags); - SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns); + SSchema *tschema = (SSchema *)(pStable->schema + pStable->numOfColumns + pStable->numOfTags); for (int32_t i = 0; i < ntags; i++) { tschema[i].colId = pStable->nextColId++; } - pStable->numOfColumns += ntags; + pStable->numOfTags += ntags; pStable->sversion++; SSdbOper oper = { @@ -840,25 +844,22 @@ static int32_t mgmtAddSuperTableTag(SSuperTableObj *pStable, SSchema schema[], i return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to add tag %s", pStable->info.tableId, schema[0].name); + mPrint("stable %s, succeed to add tag %s", pStable->info.tableId, schema[0].name); return TSDB_CODE_SUCCESS; } static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, tagName); - if (col <= 0 || col >= pStable->numOfTags) { - return TSDB_CODE_APP_ERROR; + if (col < 0) { + mError("stable:%s, drop tag, tag:%s not exist", pStable->info.tableId, tagName); + return TSDB_CODE_TAG_NOT_EXIST; } - memmove(pStable->schema + sizeof(SSchema) * col, pStable->schema + sizeof(SSchema) * (col + 1), - sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags - col - 1)); - + memmove(pStable->schema + pStable->numOfColumns + col, pStable->schema + pStable->numOfColumns + col + 1, + sizeof(SSchema) * (pStable->numOfTags - col - 1)); pStable->numOfTags--; pStable->sversion++; - int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); - pStable->schema = realloc(pStable->schema, schemaSize); - SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, @@ -871,27 +872,29 @@ static int32_t mgmtDropSuperTableTag(SSuperTableObj *pStable, char *tagName) { return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to drop tag %s", pStable->info.tableId, tagName); + mPrint("stable %s, succeed to drop tag %s", pStable->info.tableId, tagName); return TSDB_CODE_SUCCESS; } static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTagName, char *newTagName) { int32_t col = mgmtFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { - // Tag name does not exist - mError("table:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName); - return TSDB_CODE_INVALID_MSG_TYPE; + mError("stable:%s, failed to modify table tag, oldName: %s, newName: %s", pStable->info.tableId, oldTagName, newTagName); + return TSDB_CODE_TAG_NOT_EXIST; } // int32_t rowSize = 0; uint32_t len = strlen(newTagName); + if (len >= TSDB_COL_NAME_LEN) { + return TSDB_CODE_COL_NAME_TOO_LONG; + } - if (col >= pStable->numOfTags || len >= TSDB_COL_NAME_LEN || mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { - return TSDB_CODE_APP_ERROR; + if (mgmtFindSuperTableTagIndex(pStable, newTagName) >= 0) { + return TSDB_CODE_TAG_ALREAY_EXIST; } // update - SSchema *schema = (SSchema *) (pStable->schema + (pStable->numOfColumns + col) * sizeof(SSchema)); + SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col); strncpy(schema->name, newTagName, TSDB_COL_NAME_LEN); SSdbOper oper = { @@ -906,15 +909,15 @@ static int32_t mgmtModifySuperTableTagName(SSuperTableObj *pStable, char *oldTag return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName); + mPrint("stable %s, succeed to modify tag %s to %s", pStable->info.tableId, oldTagName, newTagName); return TSDB_CODE_SUCCESS; } static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) { SSchema *schema = (SSchema *) pStable->schema; - for (int32_t i = 0; i < pStable->numOfColumns; i++) { - if (strcasecmp(schema[i].name, colName) == 0) { - return i; + for (int32_t col = 0; col < pStable->numOfColumns; col++) { + if (strcasecmp(schema[col].name, colName) == 0) { + return col; } } @@ -923,21 +926,28 @@ static int32_t mgmtFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colN static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSchema schema[], int32_t ncols) { if (ncols <= 0) { + mError("stable:%s, add column, ncols:%d <= 0", pStable->info.tableId); return TSDB_CODE_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mgmtFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { - return TSDB_CODE_APP_ERROR; + mError("stable:%s, add column, column:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_FIELD_ALREAY_EXIST; + } + + if (mgmtFindSuperTableTagIndex(pStable, schema[i].name) > 0) { + mError("stable:%s, add column, tag:%s already exist", pStable->info.tableId, schema[i].name); + return TSDB_CODE_TAG_ALREAY_EXIST; } } int32_t schemaSize = sizeof(SSchema) * (pStable->numOfTags + pStable->numOfColumns); pStable->schema = realloc(pStable->schema, schemaSize + sizeof(SSchema) * ncols); - memmove(pStable->schema + sizeof(SSchema) * (pStable->numOfColumns + ncols), - pStable->schema + sizeof(SSchema) * pStable->numOfColumns, sizeof(SSchema) * pStable->numOfTags); - memcpy(pStable->schema + sizeof(SSchema) * pStable->numOfColumns, schema, sizeof(SSchema) * ncols); + memmove(pStable->schema + pStable->numOfColumns + ncols, pStable->schema + pStable->numOfColumns, + sizeof(SSchema) * pStable->numOfTags); + memcpy(pStable->schema + pStable->numOfColumns, schema, sizeof(SSchema) * ncols); SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns); for (int32_t i = 0; i < ncols; i++) { @@ -965,17 +975,18 @@ static int32_t mgmtAddSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, SSc return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to add column", pStable->info.tableId); + mPrint("stable %s, succeed to add column", pStable->info.tableId); return TSDB_CODE_SUCCESS; } static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, char *colName) { int32_t col = mgmtFindSuperTableColumnIndex(pStable, colName); - if (col < 0) { - return TSDB_CODE_APP_ERROR; + if (col <= 0) { + mError("stable:%s, drop column, column:%s not exist", pStable->info.tableId, colName); + return TSDB_CODE_FIELD_NOT_EXIST; } - memmove(pStable->schema + sizeof(SSchema) * col, pStable->schema + sizeof(SSchema) * (col + 1), + memmove(pStable->schema + col, pStable->schema + col + 1, sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags - col - 1)); pStable->numOfColumns--; @@ -1002,7 +1013,7 @@ static int32_t mgmtDropSuperTableColumn(SDbObj *pDb, SSuperTableObj *pStable, ch return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to delete column", pStable->info.tableId); + mPrint("stable %s, succeed to delete column", pStable->info.tableId); return TSDB_CODE_SUCCESS; } @@ -1490,9 +1501,9 @@ static int32_t mgmtModifyChildTableTagValue(SChildTableObj *pTable, char *tagNam static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { SSchema *schema = (SSchema *) pTable->schema; - for (int32_t i = 0; i < pTable->numOfColumns; i++) { - if (strcasecmp(schema[i].name, colName) == 0) { - return i; + for (int32_t col = 0; col < pTable->numOfColumns; col++) { + if (strcasecmp(schema[col].name, colName) == 0) { + return col; } } @@ -1501,21 +1512,23 @@ static int32_t mgmtFindNormalTableColumnIndex(SChildTableObj *pTable, char *colN static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSchema schema[], int32_t ncols) { if (ncols <= 0) { + mError("table:%s, add column, ncols:%d <= 0", pTable->info.tableId); return TSDB_CODE_APP_ERROR; } for (int32_t i = 0; i < ncols; i++) { if (mgmtFindNormalTableColumnIndex(pTable, schema[i].name) > 0) { - return TSDB_CODE_APP_ERROR; + mError("table:%s, add column, column:%s already exist", pTable->info.tableId, schema[i].name); + return TSDB_CODE_FIELD_ALREAY_EXIST; } } int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); pTable->schema = realloc(pTable->schema, schemaSize + sizeof(SSchema) * ncols); - memcpy(pTable->schema + schemaSize, schema, sizeof(SSchema) * ncols); + memcpy(pTable->schema + pTable->numOfColumns, schema, sizeof(SSchema) * ncols); - SSchema *tschema = (SSchema *) (pTable->schema + sizeof(SSchema) * pTable->numOfColumns); + SSchema *tschema = (SSchema *) (pTable->schema + pTable->numOfColumns); for (int32_t i = 0; i < ncols; i++) { tschema[i].colId = pTable->nextColId++; } @@ -1525,7 +1538,7 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc SAcctObj *pAcct = mgmtGetAcct(pDb->acct); if (pAcct != NULL) { - pAcct->acctInfo.numOfTimeSeries += ncols; + pAcct->acctInfo.numOfTimeSeries += ncols; mgmtDecAcctRef(pAcct); } @@ -1547,13 +1560,12 @@ static int32_t mgmtAddNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, SSc static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, char *colName) { int32_t col = mgmtFindNormalTableColumnIndex(pTable, colName); - if (col < 0) { - return TSDB_CODE_APP_ERROR; + if (col <= 0) { + mError("table:%s, drop column, column:%s not exist", pTable->info.tableId, colName); + return TSDB_CODE_FIELD_NOT_EXIST; } - memmove(pTable->schema + sizeof(SSchema) * col, pTable->schema + sizeof(SSchema) * (col + 1), - sizeof(SSchema) * (pTable->numOfColumns - col - 1)); - + memmove(pTable->schema + col, pTable->schema + col + 1, sizeof(SSchema) * (pTable->numOfColumns - col - 1)); pTable->numOfColumns--; pTable->sversion++; @@ -1575,7 +1587,7 @@ static int32_t mgmtDropNormalTableColumn(SDbObj *pDb, SChildTableObj *pTable, ch return TSDB_CODE_SDB_ERROR; } - mPrint("table %s, succeed to add column %s", pTable->info.tableId, colName); + mPrint("table %s, succeed to drop column %s", pTable->info.tableId, colName); return TSDB_CODE_SUCCESS; } @@ -2096,7 +2108,8 @@ static void mgmtProcessAlterTableMsg(SQueuedMsg *pMsg) { return; } - pAlter->numOfCols = htons(pAlter->numOfCols); + pAlter->type = htons(pAlter->type); + if (pAlter->numOfCols > 2) { mError("table:%s, error numOfCols:%d in alter table", pAlter->tableId, pAlter->numOfCols); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_APP_ERROR); From 4dcc0cf27b04e6e1131c83576e44b76ca0ec0f80 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sun, 26 Apr 2020 10:22:41 +0800 Subject: [PATCH 52/63] [td-197] fix error in unit test. --- src/query/src/queryExecutor.c | 9 --------- src/query/tests/astTest.cpp | 6 ++---- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 75c1b9736b..2143742e24 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -4338,15 +4338,6 @@ static UNUSED_FUNC bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, STable return false; } -static UNUSED_FUNC bool doCheckWithPrevQueryRange(SQuery *pQuery, TSKEY nextKey) { - if ((nextKey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (nextKey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - return false; - } - - return true; -} - static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index c926c9e7b9..6a78cfbe53 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -556,8 +556,7 @@ void exprSerializeTest1() { ASSERT_TRUE(size > 0); char* b = tbufGetData(&buf, false); - tExprNode* p2 = NULL; - exprTreeFromBinary(b, size, &p2); + tExprNode* p2 = exprTreeFromBinary(b, size); ASSERT_EQ(p1->nodeType, p2->nodeType); ASSERT_EQ(p2->_node.optr, p1->_node.optr); @@ -593,8 +592,7 @@ void exprSerializeTest2() { ASSERT_TRUE(size > 0); char* b = tbufGetData(&buf, false); - tExprNode* p2 = NULL; - exprTreeFromBinary(b, size, &p2); + tExprNode* p2 = exprTreeFromBinary(b, size); ASSERT_EQ(p1->nodeType, p2->nodeType); ASSERT_EQ(p2->_node.optr, p1->_node.optr); From 1f24428f3a56ceb42af5d46bb380a2529c1025c9 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sun, 26 Apr 2020 10:25:33 +0800 Subject: [PATCH 53/63] [td-197] fix error in unit test. --- src/client/src/tscUtil.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 698fa5ffe1..6b8b2b38b4 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -940,8 +940,6 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { if (pInfo->pArithExprInfo != NULL) { tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); - assert(0); -// tfree(pInfo->pArithExprInfo->colList); } } From bb8b91f25b050885c9f73eb053c9b8d6d8ffc97b Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Sun, 26 Apr 2020 11:00:54 +0800 Subject: [PATCH 54/63] [td-197] fix bug in taosmsg.h --- src/client/src/tscProfile.c | 11 ++++++----- src/inc/taosmsg.h | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index e5c3a3b7f6..61fc1a19c8 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -209,15 +209,15 @@ void tscKillStream(STscObj *pObj, uint32_t killId) { } char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj) { - SQqueryList *pQList = (SQqueryList *)pMsg; char * pMax = pMsg + TSDB_PAYLOAD_SIZE - 256; - - SQueryDesc *pQdesc = pQList->qdesc; + + SQqueryList *pQList = (SQqueryList *)pMsg; pQList->numOfQueries = 0; + SQueryDesc *pQdesc = (SQueryDesc*)(pMsg + sizeof(SQqueryList)); + // We extract the lock to tscBuildHeartBeatMsg function. /* pthread_mutex_lock (&pObj->mutex); */ - pMsg += sizeof(SQqueryList); SSqlObj *pSql = pObj->sqlList; while (pSql) { @@ -244,8 +244,9 @@ char *tscBuildQueryStreamDesc(char *pMsg, STscObj *pObj) { } SStreamList *pSList = (SStreamList *)pMsg; - SStreamDesc *pSdesc = pSList->sdesc; pSList->numOfStreams = 0; + + SStreamDesc *pSdesc = (SStreamDesc*) (pMsg + sizeof(SStreamList)); pMsg += sizeof(SStreamList); SSqlStream *pStream = pObj->streamList; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 6ff522ae43..4573e3a6e8 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -762,14 +762,14 @@ typedef struct { } SMDCfgDnodeMsg, SCMCfgDnodeMsg; typedef struct { - char sql[TSDB_SHOW_SQL_LEN + 1]; + char sql[TSDB_SHOW_SQL_LEN]; uint32_t queryId; int64_t useconds; int64_t stime; } SQueryDesc; typedef struct { - char sql[TSDB_SHOW_SQL_LEN + 1]; + char sql[TSDB_SHOW_SQL_LEN]; uint32_t streamId; int64_t num; // number of computing/cycles int64_t useconds; @@ -781,12 +781,10 @@ typedef struct { typedef struct { int32_t numOfQueries; - SQueryDesc qdesc[]; } SQqueryList; typedef struct { int32_t numOfStreams; - SStreamDesc sdesc[]; } SStreamList; typedef struct { From 7e00527838d5917383ef80a18a35ff262e9cc08b Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 26 Apr 2020 11:41:18 +0800 Subject: [PATCH 55/63] add table lastKey --- src/tsdb/inc/tsdbMain.h | 3 +++ src/tsdb/src/tsdbMain.c | 39 +++++++++++++++++++++++++++++++++++++++ src/tsdb/src/tsdbMeta.c | 1 + 3 files changed, 43 insertions(+) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5fd82956f7..7dcf6b40c2 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -74,9 +74,12 @@ typedef struct STable { void * pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index void * eventHandler; // TODO void * streamHandler; // TODO + TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure struct STable *next; // TODO: remove the next } STable; +#define TSDB_GET_TABLE_LAST_KEY(pTable) ((pTable)->lastKey) + void * tsdbEncodeTable(STable *pTable, int *contLen); STable *tsdbDecodeTable(void *cont, int contLen); void tsdbFreeEncode(void *cont); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 289ad0f0d0..e28d8f47bc 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -163,6 +163,34 @@ int32_t tsdbDropRepo(TsdbRepoT *repo) { return 0; } +static int tsdbRestoreInfo(STsdbRepo *pRepo) { + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + SFileGroup *pFGroup = NULL; + + SFileGroupIter iter; + SRWHelper rhelper = {0}; + + if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; + tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC); + while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { + if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; + for (int i = 0; i < pRepo->config.maxTables; i++) { + STable * pTable = pMeta->tables[i]; + SCompIdx *pIdx = rhelper.pCompIdx[i]; + + if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; + } + } + + tsdbDestroyHelper(&rhelper); + return 0; + +_err: + tsdbDestroyHelper(&rhelper); + return -1; +} + /** * Open an existing TSDB storage repository * @param tsdbDir the existing TSDB root directory @@ -210,6 +238,16 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { return NULL; } + // Restore key from file + if (tsdbRestoreInfo(pRepo) < 0) { + tsdbFreeCache(pRepo->tsdbCache); + tsdbFreeMeta(pRepo->tsdbMeta); + tsdbCloseFileH(pRepo->tsdbFileH); + free(pRepo->rootDir); + free(pRepo); + return NULL; + } + pRepo->state = TSDB_REPO_STATE_ACTIVE; return (TsdbRepoT *)pRepo; @@ -755,6 +793,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable tSkipListPut(pTable->mem->pData, pNode); if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; + if (key > pTable->lastKey) pTable->lastKey = key; pTable->mem->numOfPoints = tSkipListGetSize(pTable->mem->pData); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 04863d1ef2..9b606fa50a 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -311,6 +311,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) { table->tableId = pCfg->tableId; table->name = strdup(pCfg->name); + table->lastKey = 0; if (IS_CREATE_STABLE(pCfg)) { // TSDB_CHILD_TABLE table->type = TSDB_CHILD_TABLE; table->superUid = pCfg->superUid; From c588c5980ae69602ea75a7e843b796933d620124 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Sun, 26 Apr 2020 11:47:28 +0800 Subject: [PATCH 56/63] fix compile warning --- src/tsdb/src/tsdbMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index e28d8f47bc..fe03c8335e 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -177,7 +177,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; for (int i = 0; i < pRepo->config.maxTables; i++) { STable * pTable = pMeta->tables[i]; - SCompIdx *pIdx = rhelper.pCompIdx[i]; + SCompIdx *pIdx = &rhelper.pCompIdx[i]; if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; } From b13c6c029c937d4260fd6cf856ce3d30b9616962 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 26 Apr 2020 13:46:03 +0800 Subject: [PATCH 57/63] [TD-183] add db create options --- src/client/src/tscAsync.c | 2 +- src/client/src/tscSQLParser.c | 51 +++----- src/client/src/tscSql.c | 2 +- src/common/inc/tglobal.h | 8 +- src/common/src/tglobal.c | 158 ++++++++++++++---------- src/dnode/src/dnodeMgmt.c | 7 +- src/inc/taosdef.h | 72 ++++++----- src/inc/taosmsg.h | 49 ++++---- src/inc/tsdb.h | 12 +- src/mnode/inc/mgmtDef.h | 7 +- src/mnode/src/mgmtDb.c | 134 ++++++++++++-------- src/mnode/src/mgmtVgroup.c | 16 ++- src/tsdb/inc/tsdbMain.h | 3 - src/tsdb/src/tsdbCache.c | 3 + src/tsdb/src/tsdbMain.c | 29 +---- src/vnode/src/vnodeMain.c | 102 +++++++++------ tests/script/general/http/testSuite.sim | 6 +- 17 files changed, 356 insertions(+), 305 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b954db0734..3dbf74eaa7 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -46,7 +46,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const pSql->signature = pSql; pSql->param = param; pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_REPLICA_MAX_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; pSql->fp = fp; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 8d6d9baca3..b1cfbcedcf 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4800,16 +4800,15 @@ static int32_t setTimePrecisionOption(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCrea } static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { - pMsg->blocksPerTable = htons(pCreateDb->numOfBlocksPerTable); - pMsg->compression = pCreateDb->compressionLevel; - - pMsg->commitLog = (char)pCreateDb->commitLog; - pMsg->commitTime = htonl(pCreateDb->commitTime); pMsg->maxSessions = htonl(pCreateDb->tablesPerVnode); - pMsg->cacheNumOfBlocks.fraction = pCreateDb->numOfAvgCacheBlocks; - pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); - pMsg->rowsInFileBlock = htonl(pCreateDb->rowPerFileBlock); + pMsg->cacheBlockSize = htonl(-1); + pMsg->totalBlocks = htonl(-1); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); + pMsg->commitTime = htonl(pCreateDb->commitTime); + pMsg->minRowsPerFileBlock = htonl(-1); + pMsg->maxRowsPerFileBlock = htonl(-1); + pMsg->compression = pCreateDb->compressionLevel; + pMsg->commitLog = (char)pCreateDb->commitLog; pMsg->replications = pCreateDb->replica; pMsg->ignoreExist = pCreateDb->ignoreExists; } @@ -5342,29 +5341,22 @@ int32_t doLocalQueryProcess(SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { char msg[512] = {0}; - if (pCreate->commitLog != -1 && (pCreate->commitLog < 0 || pCreate->commitLog > 2)) { + if (pCreate->commitLog != -1 && (pCreate->commitLog < TSDB_MIN_CLOG_LEVEL || pCreate->commitLog > TSDB_MAX_CLOG_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option commitLog: %d, only 0-2 allowed", pCreate->commitLog); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->replications != -1 && - (pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM)) { + (pCreate->replications < TSDB_MIN_REPLICA_NUM || pCreate->replications > TSDB_MAX_REPLICA_NUM)) { snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } int32_t val = htonl(pCreate->daysPerFile); - if (val != -1 && (val < TSDB_FILE_MIN_PARTITION_RANGE || val > TSDB_FILE_MAX_PARTITION_RANGE)) { + if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, - TSDB_FILE_MIN_PARTITION_RANGE, TSDB_FILE_MAX_PARTITION_RANGE); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - } - - val = htonl(pCreate->rowsInFileBlock); - if (val != -1 && (val < TSDB_MIN_ROWS_IN_FILEBLOCK || val > TSDB_MAX_ROWS_IN_FILEBLOCK)) { - snprintf(msg, tListLen(msg), "invalid db option rowsInFileBlock: %d valid range: [%d, %d]", val, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5376,9 +5368,9 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { } val = htonl(pCreate->maxSessions); - if (val != -1 && (val < TSDB_MIN_TABLES_PER_VNODE || val > TSDB_MAX_TABLES_PER_VNODE)) { + if (val != -1 && (val < TSDB_MIN_TABLES || val > TSDB_MAX_TABLES)) { snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, - TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + TSDB_MIN_TABLES, TSDB_MAX_TABLES); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -5388,24 +5380,17 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - if (pCreate->cacheNumOfBlocks.fraction != -1 && (pCreate->cacheNumOfBlocks.fraction < TSDB_MIN_AVG_BLOCKS || - pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option ablocks: %f valid value: [%d, %d]", - pCreate->cacheNumOfBlocks.fraction, TSDB_MIN_AVG_BLOCKS, TSDB_MAX_AVG_BLOCKS); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); - } - val = htonl(pCreate->commitTime); - if (val != -1 && (val < TSDB_MIN_COMMIT_TIME_INTERVAL || val > TSDB_MAX_COMMIT_TIME_INTERVAL)) { + if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, - TSDB_MIN_COMMIT_TIME_INTERVAL, TSDB_MAX_COMMIT_TIME_INTERVAL); + TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } if (pCreate->compression != -1 && - (pCreate->compression < TSDB_MIN_COMPRESSION_LEVEL || pCreate->compression > TSDB_MAX_COMPRESSION_LEVEL)) { + (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, - TSDB_MIN_COMPRESSION_LEVEL, TSDB_MAX_COMPRESSION_LEVEL); + TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 4320e8f81b..0eff4878bf 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -131,7 +131,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con pSql->pTscObj = pObj; pSql->signature = pSql; - pSql->maxRetry = TSDB_REPLICA_MAX_NUM; + pSql->maxRetry = TSDB_MAX_REPLICA_NUM; tsem_init(&pSql->rspSem, 0, 0); diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index f770c8d38f..69829f2ba4 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -74,11 +74,13 @@ extern int16_t tsNumOfVnodesPerCore; extern int16_t tsNumOfTotalVnodes; extern uint32_t tsPublicIpInt; -extern int32_t tsMaxCacheSize; -extern int32_t tsSessionsPerVnode; +extern int32_t tsCacheBlockSize; +extern int32_t tsTotalBlocks; +extern int32_t tsTablesPerVnode; extern int16_t tsDaysPerFile; extern int32_t tsDaysToKeep; -extern int32_t tsRowsInFileBlock; +extern int32_t tsMinRowsInFileBlock; +extern int32_t tsMaxRowsInFileBlock; extern int16_t tsCommitTime; // seconds extern int32_t tsTimePrecision; extern int16_t tsCompression; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 5aa50677c9..9124295fa0 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -85,20 +85,22 @@ int16_t tsNumOfVnodesPerCore = 8; int16_t tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM; #ifdef _TD_ARM_32_ -int32_t tsSessionsPerVnode = 100; +int32_t tsTablesPerVnode = 100; #else -int32_t tsSessionsPerVnode = 1000; +int32_t tsTablesPerVnode = TSDB_DEFAULT_TABLES; #endif -int32_t tsMaxCacheSize = 64; //64M -int16_t tsDaysPerFile = 10; -int32_t tsDaysToKeep = 3650; -int32_t tsRowsInFileBlock = 4096; -int16_t tsCommitTime = 3600; // seconds -int32_t tsTimePrecision = TSDB_TIME_PRECISION_MILLI; -int16_t tsCompression = TSDB_MAX_COMPRESSION_LEVEL; -int16_t tsCommitLog = 1; -int32_t tsReplications = TSDB_REPLICA_MIN_NUM; +int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; +int32_t tsTotalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; +int16_t tsDaysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; +int32_t tsDaysToKeep = TSDB_DEFAULT_KEEP; +int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; +int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; +int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds +int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; +int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; +int16_t tsCommitLog = TSDB_DEFAULT_CLOG_LEVEL; +int32_t tsReplications = TSDB_DEFAULT_REPLICA_NUM; /** * Change the meaning of affected rows: @@ -567,16 +569,6 @@ static void doInitGlobalConfig() { cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosInitConfigOption(cfg); - cfg.option = "ctime"; - cfg.ptr = &tsCommitTime; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 30; - cfg.maxValue = 40960; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosInitConfigOption(cfg); - cfg.option = "statusInterval"; cfg.ptr = &tsStatusInterval; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -678,32 +670,42 @@ static void doInitGlobalConfig() { taosInitConfigOption(cfg); // database configs - cfg.option = "clog"; - cfg.ptr = &tsCommitLog; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.option = "tables"; + cfg.ptr = &tsTablesPerVnode; + cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 2; + cfg.minValue = TSDB_MIN_TABLES; + cfg.maxValue = TSDB_MAX_TABLES; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "comp"; - cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.option = "cache"; + cfg.ptr = &tsCacheBlockSize; + cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 2; + cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE; + cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; + taosInitConfigOption(cfg); + + cfg.option = "blocks"; + cfg.ptr = &tsTotalBlocks; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_TOTAL_BLOCKS; + cfg.maxValue = TSDB_MAX_TOTAL_BLOCKS; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); cfg.option = "days"; cfg.ptr = &tsDaysPerFile; cfg.valType = TAOS_CFG_VTYPE_INT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 365; + cfg.minValue = TSDB_MIN_DAYS_PER_FILE; + cfg.maxValue = TSDB_MAX_DAYS_PER_FILE; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -712,8 +714,58 @@ static void doInitGlobalConfig() { cfg.ptr = &tsDaysToKeep; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 365000; + cfg.minValue = TSDB_MIN_KEEP; + cfg.maxValue = TSDB_MAX_KEEP; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "minRows"; + cfg.ptr = &tsMinRowsInFileBlock; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_MIN_ROW_FBLOCK; + cfg.maxValue = TSDB_MAX_MIN_ROW_FBLOCK; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "maxRows"; + cfg.ptr = &tsMaxRowsInFileBlock; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_MAX_ROW_FBLOCK; + cfg.maxValue = TSDB_MAX_MAX_ROW_FBLOCK; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "ctime"; + cfg.ptr = &tsCommitTime; + cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_COMMIT_TIME; + cfg.maxValue = TSDB_MAX_COMMIT_TIME; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_SECOND; + taosInitConfigOption(cfg); + + cfg.option = "comp"; + cfg.ptr = &tsCompression; + cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_COMP_LEVEL; + cfg.maxValue = TSDB_MAX_COMP_LEVEL; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "clog"; + cfg.ptr = &tsCommitLog; + cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_CLOG_LEVEL; + cfg.maxValue = TSDB_MAX_CLOG_LEVEL; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -722,38 +774,8 @@ static void doInitGlobalConfig() { cfg.ptr = &tsReplications; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 3; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "tables"; - cfg.ptr = &tsSessionsPerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES_PER_VNODE; - cfg.maxValue = TSDB_MAX_TABLES_PER_VNODE; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "cache"; - cfg.ptr = &tsMaxCacheSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosInitConfigOption(cfg); - - cfg.option = "rows"; - cfg.ptr = &tsRowsInFileBlock; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 200; - cfg.maxValue = 1048576; + cfg.minValue = TSDB_MIN_REPLICA_NUM; + cfg.maxValue = TSDB_MAX_REPLICA_NUM; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 97768ca743..a319c0bb9a 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -129,13 +129,14 @@ static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); - pCreate->cfg.maxCacheSize = htobe64(pCreate->cfg.maxCacheSize); - pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); - pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); + pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); + pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); pCreate->cfg.daysPerFile = htonl(pCreate->cfg.daysPerFile); pCreate->cfg.daysToKeep1 = htonl(pCreate->cfg.daysToKeep1); pCreate->cfg.daysToKeep2 = htonl(pCreate->cfg.daysToKeep2); pCreate->cfg.daysToKeep = htonl(pCreate->cfg.daysToKeep); + pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock); + pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock); pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); pCreate->cfg.arbitratorIp = htonl(pCreate->cfg.arbitratorIp); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 043e58cb35..1a3316cdcf 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -221,44 +221,56 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_MPEERS 5 #define TSDB_MAX_MGMT_IPS (TSDB_MAX_MPEERS+1) -#define TSDB_REPLICA_MIN_NUM 1 -#define TSDB_REPLICA_MAX_NUM 3 - #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta -//default value == 10 -#define TSDB_FILE_MIN_PARTITION_RANGE 1 //minimum partition range of vnode file in days -#define TSDB_FILE_MAX_PARTITION_RANGE 3650 //max partition range of vnode file in days - -#define TSDB_DATA_MIN_RESERVE_DAY 1 // data in db to be reserved. -#define TSDB_DATA_DEFAULT_RESERVE_DAY 3650 // ten years - -#define TSDB_MIN_COMPRESSION_LEVEL 0 -#define TSDB_MAX_COMPRESSION_LEVEL 2 - -#define TSDB_MIN_COMMIT_TIME_INTERVAL 30 -#define TSDB_MAX_COMMIT_TIME_INTERVAL 40960 - -#define TSDB_MIN_ROWS_IN_FILEBLOCK 200 -#define TSDB_MAX_ROWS_IN_FILEBLOCK 500000 - #define TSDB_MIN_CACHE_BLOCK_SIZE 1 #define TSDB_MAX_CACHE_BLOCK_SIZE 1000000 +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 -#define TSDB_MIN_CACHE_BLOCKS 100 -#define TSDB_MAX_CACHE_BLOCKS 409600 +#define TSDB_MIN_TOTAL_BLOCKS 2 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 2 -#define TSDB_MIN_AVG_BLOCKS 2 -#define TSDB_MAX_AVG_BLOCKS 2048 -#define TSDB_DEFAULT_AVG_BLOCKS 4 +#define TSDB_MIN_TABLES 4 +#define TSDB_MAX_TABLES 200000 +#define TSDB_DEFAULT_TABLES 1000 -/* - * There is a bug in function taosAllocateId. - * When "create database tables 1" is executed, the wrong sid is assigned, so the minimum value is set to 2. - */ -#define TSDB_MIN_TABLES_PER_VNODE 2 -#define TSDB_MAX_TABLES_PER_VNODE 220000 +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 + +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years + +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 + +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 + +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 + +#define TSDB_MIN_PRECISION TSDB_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI + +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 + +#define TSDB_MIN_CLOG_LEVEL 0 +#define TSDB_MAX_CLOG_LEVEL 2 +#define TSDB_DEFAULT_CLOG_LEVEL 2 + +#define TSDB_MIN_REPLICA_NUM 1 +#define TSDB_MAX_REPLICA_NUM 3 +#define TSDB_DEFAULT_REPLICA_NUM 1 #define TSDB_MAX_JOIN_TABLE_NUM 5 #define TSDB_MAX_UNION_CLAUSE 5 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 8e6e762017..54523196dd 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -517,27 +517,21 @@ typedef struct { typedef struct { char acct[TSDB_USER_LEN + 1]; char db[TSDB_DB_NAME_LEN + 1]; - uint32_t vgId; int32_t maxSessions; - int32_t cacheBlockSize; - union { - int32_t totalBlocks; - float fraction; - } cacheNumOfBlocks; - int32_t daysPerFile; - int32_t daysToKeep1; - int32_t daysToKeep2; - int32_t daysToKeep; - int32_t commitTime; - int32_t rowsInFileBlock; - int16_t blocksPerTable; - int8_t compression; - int8_t commitLog; - int8_t replications; - int8_t repStrategy; - int8_t loadLatest; // load into mem or not - uint8_t precision; // time resolution - int8_t ignoreExist; + int32_t cacheBlockSize; //MB + int32_t totalBlocks; + int32_t daysPerFile; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t daysToKeep; + int32_t commitTime; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; + int8_t compression; + int8_t commitLog; + int8_t replications; + uint8_t precision; // time resolution + int8_t ignoreExist; } SCMCreateDbMsg, SCMAlterDbMsg; typedef struct { @@ -605,20 +599,21 @@ typedef struct { typedef struct { uint32_t vgId; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; - int64_t maxCacheSize; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; int32_t daysPerFile; int32_t daysToKeep; int32_t daysToKeep1; int32_t daysToKeep2; + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; int32_t commitTime; - uint8_t precision; // time resolution + int8_t precision; int8_t compression; - int8_t wals; int8_t commitLog; int8_t replications; + int8_t wals; int8_t quorum; uint32_t arbitratorIp; int8_t reserved[16]; @@ -653,7 +648,7 @@ typedef struct SCMSTableVgroupMsg { typedef struct { int32_t vgId; int8_t numOfIps; - SIpAddr ipAddr[TSDB_REPLICA_MAX_NUM]; + SIpAddr ipAddr[TSDB_MAX_REPLICA_NUM]; } SCMVgroupInfo; typedef struct { @@ -697,7 +692,7 @@ typedef struct { } SVnodeDesc; typedef struct { - SVnodeDesc vpeerDesc[TSDB_REPLICA_MAX_NUM]; + SVnodeDesc vpeerDesc[TSDB_MAX_REPLICA_NUM]; int16_t index; // used locally int32_t vgId; int32_t numOfSids; diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index e5c884c819..38ded5cf10 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -45,15 +45,19 @@ typedef struct { // --------- TSDB REPOSITORY CONFIGURATION DEFINITION typedef struct { - int8_t precision; - int8_t compression; int32_t tsdbId; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; // maximum number of tables this repository can have int32_t daysPerFile; // day per file sharding policy + int32_t keep; // day of data to keep + int32_t keep1; + int32_t keep2; int32_t minRowsPerFileBlock; // minimum rows per file block int32_t maxRowsPerFileBlock; // maximum rows per file block - int32_t keep; // day of data to keep - int64_t maxCacheSize; // maximum cache size this TSDB can use + int32_t commitTime; + int8_t precision; + int8_t compression; } STsdbCfg; void tsdbSetDefaultCfg(STsdbCfg *pCfg); diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index 6cbfc72e62..f91a8d6bb4 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -143,14 +143,15 @@ typedef struct SVgObj { } SVgObj; typedef struct { - int64_t maxCacheSize; + int32_t cacheBlockSize; + int32_t totalBlocks; int32_t maxTables; int32_t daysPerFile; int32_t daysToKeep; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t minRowsPerFileBlock; // minimum rows per file block - int32_t maxRowsPerFileBlock; // maximum rows per file block + int32_t minRowsPerFileBlock; + int32_t maxRowsPerFileBlock; int32_t commitTime; int8_t precision; int8_t compression; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index b3424ffd63..844d40b605 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -177,78 +177,81 @@ SDbObj *mgmtGetDbByTableId(char *tableId) { } static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) { - if (pCfg->maxCacheSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_BLOCK_SIZE) { - mError("invalid db option maxCacheSize:%d valid range: [%d, %d]", pCfg->maxCacheSize, TSDB_MIN_CACHE_BLOCK_SIZE, + if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { + mError("invalid db option cacheBlockSize:%d valid range: [%d, %d]", pCfg->cacheBlockSize, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); } - if (pCfg->maxTables < TSDB_MIN_TABLES_PER_VNODE || pCfg->maxTables > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES_PER_VNODE, - TSDB_MAX_TABLES_PER_VNODE); + if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { + mError("invalid db option totalBlocks:%d valid range: [%d, %d]", pCfg->totalBlocks, TSDB_MIN_TOTAL_BLOCKS, + TSDB_MAX_TOTAL_BLOCKS); + } + + if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) { + mError("invalid db option maxTables:%d valid range: [%d, %d]", pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->daysPerFile < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysPerFile > TSDB_FILE_MAX_PARTITION_RANGE) { - mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_FILE_MIN_PARTITION_RANGE, - TSDB_FILE_MAX_PARTITION_RANGE); + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { + mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, + TSDB_MAX_DAYS_PER_FILE); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->daysToKeep1 < TSDB_FILE_MIN_PARTITION_RANGE || pCfg->daysToKeep1 < pCfg->daysPerFile) { - mError("invalid db option daystokeep:%d", pCfg->daysToKeep); + if (pCfg->daysToKeep < TSDB_MIN_KEEP || pCfg->daysToKeep > TSDB_MAX_KEEP) { + mError("invalid db option daysToKeep:%d", pCfg->daysToKeep); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->daysToKeep2 > pCfg->daysToKeep || pCfg->daysToKeep2 < pCfg->daysToKeep1) { - mError("invalid db option daystokeep1:%d, daystokeep2:%d, daystokeep:%d", pCfg->daysToKeep1, - pCfg->daysToKeep2, pCfg->daysToKeep); + if (pCfg->daysToKeep < pCfg->daysPerFile) { + mError("invalid db option daysToKeep:%d daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile); return TSDB_CODE_INVALID_OPTION; } - - if (pCfg->minRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { + + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { mError("invalid db option minRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->minRowsPerFileBlock, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_ROWS_IN_FILEBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_ROWS_IN_FILEBLOCK) { + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { mError("invalid db option maxRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->maxRowsPerFileBlock, - TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); + TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->maxRowsPerFileBlock < pCfg->minRowsPerFileBlock) { + if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { mError("invalid db option minRowsPerFileBlock:%d maxRowsPerFileBlock:%d", pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME_INTERVAL || pCfg->commitTime > TSDB_MAX_COMMIT_TIME_INTERVAL) { - mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME_INTERVAL, - TSDB_MAX_COMMIT_TIME_INTERVAL); + if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { + mError("invalid db option commitTime:%d valid range: [%d, %d]", pCfg->commitTime, TSDB_MIN_COMMIT_TIME, + TSDB_MAX_COMMIT_TIME); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->precision != TSDB_TIME_PRECISION_MILLI && pCfg->precision != TSDB_TIME_PRECISION_MICRO) { - mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_TIME_PRECISION_MILLI, - TSDB_TIME_PRECISION_MICRO); + if (pCfg->precision != TSDB_MIN_PRECISION && pCfg->precision != TSDB_MAX_PRECISION) { + mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_MIN_PRECISION, + TSDB_MAX_PRECISION); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->compression < TSDB_MIN_COMPRESSION_LEVEL || pCfg->compression > TSDB_MAX_COMPRESSION_LEVEL) { - mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMPRESSION_LEVEL, - TSDB_MAX_COMPRESSION_LEVEL); + if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { + mError("invalid db option compression:%d valid range: [%d, %d]", pCfg->compression, TSDB_MIN_COMP_LEVEL, + TSDB_MAX_COMP_LEVEL); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->commitLog < 0 || pCfg->commitLog > 2) { + if (pCfg->commitLog < TSDB_MIN_CLOG_LEVEL || pCfg->commitLog > TSDB_MAX_CLOG_LEVEL) { mError("invalid db option commitLog:%d, only 0-2 allowed", pCfg->commitLog); return TSDB_CODE_INVALID_OPTION; } - if (pCfg->replications < TSDB_REPLICA_MIN_NUM || pCfg->replications > TSDB_REPLICA_MAX_NUM) { - mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_REPLICA_MIN_NUM, - TSDB_REPLICA_MAX_NUM); + if (pCfg->replications < TSDB_MIN_REPLICA_NUM || pCfg->replications > TSDB_MAX_REPLICA_NUM) { + mError("invalid db option replications:%d valid range: [%d, %d]", pCfg->replications, TSDB_MIN_REPLICA_NUM, + TSDB_MAX_REPLICA_NUM); return TSDB_CODE_INVALID_OPTION; } @@ -256,14 +259,15 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) { } static void mgmtSetDefaultDbCfg(SDbCfg *pCfg) { - if (pCfg->maxCacheSize < 0) pCfg->maxCacheSize = tsMaxCacheSize; - if (pCfg->maxTables < 0) pCfg->maxTables = tsSessionsPerVnode; + if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = tsCacheBlockSize; + if (pCfg->totalBlocks < 0) pCfg->totalBlocks = tsTotalBlocks; + if (pCfg->maxTables < 0) pCfg->maxTables = tsTablesPerVnode; if (pCfg->daysPerFile < 0) pCfg->daysPerFile = tsDaysPerFile; if (pCfg->daysToKeep < 0) pCfg->daysToKeep = tsDaysToKeep; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep; if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep; - if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsRowsInFileBlock; - if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = pCfg->minRowsPerFileBlock * 2; + if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = tsMinRowsInFileBlock; + if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = tsMaxRowsInFileBlock; if (pCfg->commitTime < 0) pCfg->commitTime = tsCommitTime; if (pCfg->precision < 0) pCfg->precision = tsTimePrecision; if (pCfg->compression < 0) pCfg->compression = tsCompression; @@ -293,14 +297,15 @@ static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate) { strncpy(pDb->acct, pAcct->user, TSDB_USER_LEN); pDb->createdTime = taosGetTimestampMs(); pDb->cfg = (SDbCfg) { - .maxCacheSize = 64,//(int64_t)pCreate->cacheBlockSize * pCreate->cacheNumOfBlocks.totalBlocks, + .cacheBlockSize = pCreate->cacheBlockSize, + .totalBlocks = pCreate->totalBlocks, .maxTables = pCreate->maxSessions, .daysPerFile = pCreate->daysPerFile, .daysToKeep = pCreate->daysToKeep, .daysToKeep1 = pCreate->daysToKeep1, .daysToKeep2 = pCreate->daysToKeep2, - .minRowsPerFileBlock = pCreate->rowsInFileBlock * 1, - .maxRowsPerFileBlock = pCreate->rowsInFileBlock * 2, + .minRowsPerFileBlock = pCreate->maxRowsPerFileBlock, + .maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock, .commitTime = pCreate->commitTime, .precision = pCreate->precision, .compression = pCreate->compression, @@ -459,13 +464,25 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "rows"); + strcpy(pSchema[cols].name, "cache(MB)"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "blocks"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "minrows"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cache(Mb)"); + strcpy(pSchema[cols].name, "maxrows"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -492,7 +509,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) pShow->bytes[cols] = 3; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "time precision"); + strcpy(pSchema[cols].name, "precision"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -583,14 +600,22 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * *(int32_t *)pWrite = pDb->cfg.maxTables; // table num can be created should minus 1 cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.totalBlocks; + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxCacheSize; + *(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock; cols++; - + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int32_t *)pWrite = pDb->cfg.commitTime; cols++; @@ -664,14 +689,15 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { SCMCreateDbMsg *pCreate = pMsg->pCont; pCreate->maxSessions = htonl(pCreate->maxSessions); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); + pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->daysPerFile = htonl(pCreate->daysPerFile); pCreate->daysToKeep = htonl(pCreate->daysToKeep); pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); pCreate->commitTime = htonl(pCreate->commitTime); - pCreate->blocksPerTable = htons(pCreate->blocksPerTable); - pCreate->rowsInFileBlock = htonl(pCreate->rowsInFileBlock); - + pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); + pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + int32_t code; if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) { code = TSDB_CODE_GRANT_EXPIRED; @@ -688,9 +714,9 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { } static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { - SDbCfg newCfg = pDb->cfg; - int32_t daysToKeep = htonl(pAlter->daysToKeep); - int32_t maxTables = htonl(pAlter->maxSessions); + SDbCfg newCfg = pDb->cfg; + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t maxTables = htonl(pAlter->maxSessions); int8_t replications = pAlter->replications; terrno = TSDB_CODE_SUCCESS; @@ -702,8 +728,8 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { if (replications > 0 && replications != pDb->cfg.replications) { mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); - if (replications < TSDB_REPLICA_MIN_NUM || replications > TSDB_REPLICA_MAX_NUM) { - mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); + if (replications < TSDB_MIN_REPLICA_NUM || replications > TSDB_MAX_REPLICA_NUM) { + mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM); terrno = TSDB_CODE_INVALID_OPTION; } newCfg.replications = replications; @@ -711,8 +737,8 @@ static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { if (maxTables > 0 && maxTables != pDb->cfg.maxTables) { mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables); - if (maxTables < TSDB_MIN_TABLES_PER_VNODE || maxTables > TSDB_MAX_TABLES_PER_VNODE) { - mError("invalid db option tables: %d valid range: %d--%d", maxTables, TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); + if (maxTables < TSDB_MIN_TABLES || maxTables > TSDB_MAX_TABLES) { + mError("invalid db option tables: %d valid range: %d--%d", maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); terrno = TSDB_CODE_INVALID_OPTION; } if (maxTables < pDb->cfg.maxTables) { diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index e053ea66f5..feedf5fdc3 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -535,23 +535,21 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SMDVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); + pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); + pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks); pCfg->maxTables = htonl(pDb->cfg.maxTables); - pCfg->maxCacheSize = htobe64(pDb->cfg.maxCacheSize); - pCfg->maxCacheSize = htobe64(-1); //TODO - pCfg->minRowsPerFileBlock = htonl(-1); //TODO - pCfg->maxRowsPerFileBlock = htonl(-1); //TODO pCfg->daysPerFile = htonl(pDb->cfg.daysPerFile); - pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); - pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); pCfg->daysToKeep = htonl(pDb->cfg.daysToKeep); - pCfg->daysToKeep = htonl(-1); //TODO + pCfg->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); + pCfg->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); + pCfg->minRowsPerFileBlock = htonl(pDb->cfg.minRowsPerFileBlock); + pCfg->maxRowsPerFileBlock = htonl(pDb->cfg.maxRowsPerFileBlock); pCfg->commitTime = htonl(pDb->cfg.commitTime); pCfg->precision = pDb->cfg.precision; pCfg->compression = pDb->cfg.compression; - pCfg->compression = -1; - pCfg->wals = 3; pCfg->commitLog = pDb->cfg.commitLog; pCfg->replications = (int8_t) pVgroup->numOfVnodes; + pCfg->wals = 3; pCfg->quorum = 1; SMDVnodeDesc *pNodes = pVnode->nodes; diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5fd82956f7..d4ed7a6384 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -129,9 +129,6 @@ STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); STable *tsdbGetTableByUid(STsdbMeta *pMeta, int64_t uid); char * getTupleKey(const void *data); -// ------------------------------ TSDB CACHE INTERFACES ------------------------------ -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 * 1024 * 1024 /* 16M */ - typedef struct { int blockId; int offset; diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c index 942afd70a6..3e241773ed 100644 --- a/src/tsdb/src/tsdbCache.c +++ b/src/tsdb/src/tsdbCache.c @@ -26,6 +26,9 @@ STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) { if (pCache == NULL) return NULL; if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; + cacheBlockSize *= (1024 * 1024); + + if (maxBytes < 0) maxBytes = cacheBlockSize * TSDB_DEFAULT_TOTAL_BLOCKS; pCache->maxBytes = maxBytes; pCache->cacheBlockSize = cacheBlockSize; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 289ad0f0d0..ea1bc84d65 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -1,4 +1,5 @@ #include "os.h" +#include "taosdef.h" #include "tulog.h" #include "talgo.h" #include "tsdb.h" @@ -11,24 +12,6 @@ #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) #define TSDB_MIN_ID 0 #define TSDB_MAX_ID INT_MAX -#define TSDB_MIN_TABLES 4 -#define TSDB_MAX_TABLES 100000 -#define TSDB_DEFAULT_TABLES 1000 -#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_DAYS_PER_FILE 1 -#define TSDB_MAX_DAYS_PER_FILE 60 -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_DEFAULT_KEEP 3650 -#define TSDB_MIN_KEEP 1 -#define TSDB_MAX_KEEP INT_MAX -#define TSDB_DEFAULT_CACHE_SIZE (16 * 1024 * 1024) // 16M -#define TSDB_MIN_CACHE_SIZE (4 * 1024 * 1024) // 4M -#define TSDB_MAX_CACHE_SIZE (1024 * 1024 * 1024) // 1G #define TSDB_CFG_FILE_NAME "CONFIG" #define TSDB_DATA_DIR_NAME "data" @@ -70,7 +53,6 @@ void tsdbSetDefaultCfg(STsdbCfg *pCfg) { pCfg->minRowsPerFileBlock = -1; pCfg->maxRowsPerFileBlock = -1; pCfg->keep = -1; - pCfg->maxCacheSize = -1; pCfg->compression = TWO_STAGE_COMP; } @@ -192,7 +174,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) { return NULL; } - pRepo->tsdbCache = tsdbInitCache(pRepo->config.maxCacheSize, -1, (TsdbRepoT *)pRepo); + pRepo->tsdbCache = tsdbInitCache(-1, -1, (TsdbRepoT *)pRepo); if (pRepo->tsdbCache == NULL) { tsdbFreeMeta(pRepo->tsdbMeta); free(pRepo->rootDir); @@ -612,13 +594,6 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) return -1; } - // Check maxCacheSize - if (pCfg->maxCacheSize == -1) { - pCfg->maxCacheSize = TSDB_DEFAULT_CACHE_SIZE; - } else { - if (pCfg->maxCacheSize < TSDB_MIN_CACHE_SIZE || pCfg->maxCacheSize > TSDB_MAX_CACHE_SIZE) return -1; - } - return 0; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 26791ae77c..153bdbffd8 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -104,8 +104,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; - tsdbCfg.maxCacheSize = pVnodeCfg->cfg.maxCacheSize; - + char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); code = tsdbCreateRepo(tsdbDir, &tsdbCfg, NULL); @@ -390,26 +389,27 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); - len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); + len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); + len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile); + len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); + len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1); + len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2); len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock); len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); - - len += snprintf(content + len, maxLen - len, " \"maxCacheSize\": %" PRId64 ",\n", pVnodeCfg->cfg.maxCacheSize); - + len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); + len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); + len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); len += snprintf(content + len, maxLen - len, " \"commitLog\": %d,\n", pVnodeCfg->cfg.commitLog); + len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pVnodeCfg->cfg.wals); - + len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); + uint32_t ipInt = pVnodeCfg->cfg.arbitratorIp; sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); len += snprintf(content + len, maxLen - len, " \"arbitratorIp\": \"%s\",\n", ipStr); - len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pVnodeCfg->cfg.quorum); - len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pVnodeCfg->cfg.replications); - len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); for (int32_t i = 0; i < pVnodeCfg->cfg.replications; i++) { len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", pVnodeCfg->nodes[i].nodeId); @@ -463,19 +463,19 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_OVER; } - cJSON *precision = cJSON_GetObjectItem(root, "precision"); - if (!precision || precision->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId); + cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); + if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.precision = (int8_t)precision->valueint; + pVnode->tsdbCfg.cacheBlockSize = cacheBlockSize->valueint; - cJSON *compression = cJSON_GetObjectItem(root, "compression"); - if (!compression || compression->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId); + cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); + if (!totalBlocks || totalBlocks->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, totalBlocks not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.compression = (int8_t)compression->valueint; + pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); if (!maxTables || maxTables->type != cJSON_Number) { @@ -484,13 +484,34 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxTables = maxTables->valueint; - cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); + cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysPerFile not found", pVnode, pVnode->vgId); goto PARSE_OVER; } pVnode->tsdbCfg.daysPerFile = daysPerFile->valueint; + cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); + if (!daysToKeep || daysToKeep->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep = daysToKeep->valueint; + + cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); + if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep1 not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep1 = daysToKeep1->valueint; + + cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); + if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep2 not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.keep2 = daysToKeep2->valueint; + cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, minRowsPerFileBlock not found", pVnode, pVnode->vgId); @@ -505,19 +526,26 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); - if (!daysToKeep || daysToKeep->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, daysToKeep not found", pVnode, pVnode->vgId); + cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); + if (!commitTime || commitTime->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, commitTime not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.keep = daysToKeep->valueint; + pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; - cJSON *maxCacheSize = cJSON_GetObjectItem(root, "maxCacheSize"); - if (!maxCacheSize || maxCacheSize->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, maxCacheSize not found", pVnode, pVnode->vgId); + cJSON *precision = cJSON_GetObjectItem(root, "precision"); + if (!precision || precision->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, precision not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->tsdbCfg.maxCacheSize = maxCacheSize->valueint; + pVnode->tsdbCfg.precision = (int8_t)precision->valueint; + + cJSON *compression = cJSON_GetObjectItem(root, "compression"); + if (!compression || compression->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, compression not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->tsdbCfg.compression = (int8_t)compression->valueint; cJSON *commitLog = cJSON_GetObjectItem(root, "commitLog"); if (!commitLog || commitLog->type != cJSON_Number) { @@ -534,12 +562,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { pVnode->walCfg.wals = (int8_t)wals->valueint; pVnode->walCfg.keep = 0; - cJSON *arbitratorIp = cJSON_GetObjectItem(root, "arbitratorIp"); - if (!arbitratorIp || arbitratorIp->type != cJSON_String || arbitratorIp->valuestring == NULL) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, arbitratorIp not found", pVnode, pVnode->vgId); + cJSON *replica = cJSON_GetObjectItem(root, "replica"); + if (!replica || replica->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->syncCfg.arbitratorIp = inet_addr(arbitratorIp->valuestring); + pVnode->syncCfg.replica = (int8_t)replica->valueint; cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); if (!quorum || quorum->type != cJSON_Number) { @@ -548,12 +576,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->syncCfg.quorum = (int8_t)quorum->valueint; - cJSON *replica = cJSON_GetObjectItem(root, "replica"); - if (!replica || replica->type != cJSON_Number) { - dError("pVnode:%p vgId:%d, failed to read vnode cfg, replica not found", pVnode, pVnode->vgId); + cJSON *arbitratorIp = cJSON_GetObjectItem(root, "arbitratorIp"); + if (!arbitratorIp || arbitratorIp->type != cJSON_String || arbitratorIp->valuestring == NULL) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, arbitratorIp not found", pVnode, pVnode->vgId); goto PARSE_OVER; } - pVnode->syncCfg.replica = (int8_t)replica->valueint; + pVnode->syncCfg.arbitratorIp = inet_addr(arbitratorIp->valuestring); cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index 092eebdbdf..cec8da67bb 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -1,6 +1,8 @@ run general/http/restful.sim -run general/http/restful_full.sim +run general/http/restful_insert.sim +#run general/http/restful_limit.sim +#run general/http/restful_full.sim +#run general/http/prepare.sim run general/http/telegraf.sim -run general/http/prepare.sim run general/http/grafana_bug.sim run general/http/grafana.sim From 48c0684ed58d6aa113e8b4095c9d01347907aa69 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 26 Apr 2020 15:31:39 +0800 Subject: [PATCH 58/63] add bigint bool double smallint tinyint cases. --- tests/pytest/insert/bigint.py | 69 +++++++++++++++++++++ tests/pytest/insert/bool.py | 69 +++++++++++++++++++++ tests/pytest/insert/double.py | 90 +++++++++++++++++++++++++++ tests/pytest/insert/smallint.py | 106 ++++++++++++++++++++++++++++++++ tests/pytest/insert/tinyint.py | 106 ++++++++++++++++++++++++++++++++ 5 files changed, 440 insertions(+) create mode 100644 tests/pytest/insert/bigint.py create mode 100644 tests/pytest/insert/bool.py create mode 100644 tests/pytest/insert/double.py create mode 100644 tests/pytest/insert/smallint.py create mode 100644 tests/pytest/insert/tinyint.py diff --git a/tests/pytest/insert/bigint.py b/tests/pytest/insert/bigint.py new file mode 100644 index 0000000000..214200900a --- /dev/null +++ b/tests/pytest/insert/bigint.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed bigint)') + tdSql.execute('create table tb (ts timestamp, speed bigint)') + tdLog.info("insert into tb values (now, -9223372036854770000)") + tdSql.execute("insert into tb values (now, -9223372036854770000)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, -9223372036854770000)') + tdSql.checkData(0, 1, -9223372036854770000) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1a, -9223372036854770000)") + tdSql.execute("insert into tb values (now+1a, -9223372036854770000)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, -9223372036854770000)') + tdSql.checkData(0, 1, -9223372036854770000) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2a, 9223372036854770000)") + tdSql.execute("insert into tb values (now+2a, 9223372036854770000)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 9223372036854770000)') + tdSql.checkData(0, 1, 9223372036854770000) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3a, 9223372036854770000)") + tdSql.execute("insert into tb values (now+3a, 9223372036854770000)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, 9223372036854770000)') + tdSql.checkData(0, 1, 9223372036854770000) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/bool.py b/tests/pytest/insert/bool.py new file mode 100644 index 0000000000..062563f4ab --- /dev/null +++ b/tests/pytest/insert/bool.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed bool)') + tdSql.execute('create table tb (ts timestamp, speed bool)') + tdLog.info("insert into tb values (now, true)") + tdSql.execute("insert into tb values (now, true)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, 1)") + tdSql.execute("insert into tb values (now+1m, 1)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 2)") + tdSql.execute("insert into tb values (now+2m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 0)") + tdSql.execute("insert into tb values (now+3m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, -1)") + tdSql.execute("insert into tb values (now+4m, -1)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, false)") + tdSql.execute("insert into tb values (now+5m, false)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/double.py b/tests/pytest/insert/double.py new file mode 100644 index 0000000000..19be9653c0 --- /dev/null +++ b/tests/pytest/insert/double.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed double)') + tdSql.execute('create table tb (ts timestamp, speed double)') + tdLog.info('=============== step2') + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2a, 2.85)") + tdSql.execute("insert into tb values (now+2a, 2.85)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, 2.850000000)') + tdSql.checkData(0, 1, 2.850000000) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3a, 3.4)") + tdSql.execute("insert into tb values (now+3a, 3.4)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, 3.400000000)') + tdSql.checkData(0, 1, 3.400000000) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4a, a2) -x step51") + tdSql.error("insert into tb values (now+4a, a2)1") + tdLog.info("insert into tb values (now+4a, 0)") + tdSql.execute("insert into tb values (now+4a, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 0.000000000)') + tdSql.checkData(0, 1, 0.000000000) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5a, 2a) -x step6") + tdSql.error("insert into tb values (now+5a, 2a)") + tdLog.info("insert into tb values(now+5a, 2)") + tdSql.execute("insert into tb values(now+5a, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, 2.000000000)') + tdSql.checkData(0, 1, 2.000000000) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6a, 2a'1) -x step7") + tdSql.error("insert into tb values (now+6a, 2a'1)") + tdLog.info("insert into tb values(now+6a, 2)") + tdSql.execute("insert into tb values(now+6a, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 2.000000000)') + tdSql.checkData(0, 1, 2.000000000) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/smallint.py b/tests/pytest/insert/smallint.py new file mode 100644 index 0000000000..0d85e45cfa --- /dev/null +++ b/tests/pytest/insert/smallint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed smallint)') + tdSql.execute('create table tb (ts timestamp, speed smallint)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -32768) -x step2") + tdSql.error("insert into tb values (now+1m, -32768)") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 32767)") + tdSql.execute("insert into tb values (now+2m, 32767)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 32767)') + tdSql.checkData(0, 1, 32767) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 32768) -x step4") + tdSql.error("insert into tb values (now+3m, 32768)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/tinyint.py b/tests/pytest/insert/tinyint.py new file mode 100644 index 0000000000..6d7178054e --- /dev/null +++ b/tests/pytest/insert/tinyint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed tinyint)') + tdSql.execute('create table tb (ts timestamp, speed tinyint)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -128) -x step2") + tdSql.error("insert into tb values (now+1m, -128) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 127)") + tdSql.execute("insert into tb values (now+2m, 127)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 127)') + tdSql.checkData(0, 1, 127) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 128) -x step4") + tdSql.error("insert into tb values (now+3m, 128)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 9a9468d1d4ef1695335846bcbf66595f06532243 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 26 Apr 2020 16:00:32 +0800 Subject: [PATCH 59/63] [TD-185] add db alter options --- src/client/src/tscServer.c | 5 -- src/dnode/src/dnodeMgmt.c | 1 + src/inc/taosmsg.h | 2 + src/mnode/inc/mgmtDef.h | 1 + src/mnode/src/mgmtDb.c | 106 +++++++++++++++++++++++++------------ src/mnode/src/mgmtDnode.c | 3 +- src/mnode/src/mgmtVgroup.c | 18 ++----- src/vnode/inc/vnodeInt.h | 1 + src/vnode/src/vnodeMain.c | 9 ++++ 9 files changed, 92 insertions(+), 54 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index c048714712..a9b2e12fd5 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1377,11 +1377,6 @@ int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = sizeof(SCMAlterDbMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB; - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_CLI_OUT_OF_MEMORY; - } - SCMAlterDbMsg *pAlterDbMsg = (SCMAlterDbMsg*)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); strcpy(pAlterDbMsg->db, pTableMetaInfo->name); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index a319c0bb9a..fef91c5786 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -128,6 +128,7 @@ static void dnodeCloseVnodes() { static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) { SMDCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); + pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion); pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 54523196dd..1b422af31f 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -505,6 +505,7 @@ typedef struct SRetrieveTableRsp { typedef struct { int32_t vgId; + int32_t cfgVersion; int64_t totalStorage; int64_t compStorage; int64_t pointsWritten; @@ -599,6 +600,7 @@ typedef struct { typedef struct { uint32_t vgId; + int32_t cfgVersion; int32_t cacheBlockSize; int32_t totalBlocks; int32_t maxTables; diff --git a/src/mnode/inc/mgmtDef.h b/src/mnode/inc/mgmtDef.h index f91a8d6bb4..a7e75f9777 100644 --- a/src/mnode/inc/mgmtDef.h +++ b/src/mnode/inc/mgmtDef.h @@ -164,6 +164,7 @@ typedef struct SDbObj { char name[TSDB_DB_NAME_LEN + 1]; char acct[TSDB_USER_LEN + 1]; int64_t createdTime; + int32_t cfgVersion; SDbCfg cfg; int8_t status; int8_t reserved[14]; diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 844d40b605..29d2343231 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -22,6 +22,7 @@ #include "tglobal.h" #include "ttime.h" #include "tname.h" +#include "tbalance.h" #include "mgmtDef.h" #include "mgmtLog.h" #include "mgmtAcct.h" @@ -715,39 +716,65 @@ static void mgmtProcessCreateDbMsg(SQueuedMsg *pMsg) { static SDbCfg mgmtGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { SDbCfg newCfg = pDb->cfg; - int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t cacheBlockSize = htonl(pAlter->daysToKeep); + int32_t totalBlocks = htonl(pAlter->totalBlocks); int32_t maxTables = htonl(pAlter->maxSessions); + int32_t daysToKeep = htonl(pAlter->daysToKeep); + int32_t daysToKeep1 = htonl(pAlter->daysToKeep1); + int32_t daysToKeep2 = htonl(pAlter->daysToKeep2); + int8_t compression = pAlter->compression; int8_t replications = pAlter->replications; terrno = TSDB_CODE_SUCCESS; + if (cacheBlockSize > 0 && cacheBlockSize != pDb->cfg.cacheBlockSize) { + mTrace("db:%s, cache:%d change to %d", pDb->name, pDb->cfg.cacheBlockSize, cacheBlockSize); + newCfg.cacheBlockSize = cacheBlockSize; + } + + if (totalBlocks > 0 && totalBlocks != pDb->cfg.totalBlocks) { + mTrace("db:%s, blocks:%d change to %d", pDb->name, pDb->cfg.totalBlocks, totalBlocks); + newCfg.totalBlocks = totalBlocks; + } + + if (maxTables > 0 && maxTables != pDb->cfg.maxTables) { + mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables); + newCfg.maxTables = maxTables; + if (newCfg.maxTables < pDb->cfg.maxTables) { + mTrace("db:%s, tables:%d should larger than origin:%d", pDb->name, newCfg.maxTables, pDb->cfg.maxTables); + terrno = TSDB_CODE_INVALID_OPTION; + } + } + if (daysToKeep > 0 && daysToKeep != pDb->cfg.daysToKeep) { mTrace("db:%s, daysToKeep:%d change to %d", pDb->name, pDb->cfg.daysToKeep, daysToKeep); newCfg.daysToKeep = daysToKeep; - } - - if (replications > 0 && replications != pDb->cfg.replications) { - mTrace("db:%s, replica:%d change to %d", pDb->name, pDb->cfg.replications, replications); - if (replications < TSDB_MIN_REPLICA_NUM || replications > TSDB_MAX_REPLICA_NUM) { - mError("invalid db option replica: %d valid range: %d--%d", replications, TSDB_MIN_REPLICA_NUM, TSDB_MAX_REPLICA_NUM); - terrno = TSDB_CODE_INVALID_OPTION; - } - newCfg.replications = replications; - } - - if (maxTables > 0 && maxTables != pDb->cfg.maxTables) { - mTrace("db:%s, tables:%d change to %d", pDb->name, pDb->cfg.maxTables, maxTables); - if (maxTables < TSDB_MIN_TABLES || maxTables > TSDB_MAX_TABLES) { - mError("invalid db option tables: %d valid range: %d--%d", maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); - terrno = TSDB_CODE_INVALID_OPTION; - } - if (maxTables < pDb->cfg.maxTables) { - mError("invalid db option tables: %d should larger than original:%d", maxTables, pDb->cfg.maxTables); - terrno = TSDB_CODE_INVALID_OPTION; - } - newCfg.maxTables = maxTables; } + if (daysToKeep1 > 0 && daysToKeep1 != pDb->cfg.daysToKeep1) { + mTrace("db:%s, daysToKeep1:%d change to %d", pDb->name, pDb->cfg.daysToKeep1, daysToKeep1); + newCfg.daysToKeep1 = daysToKeep1; + } + + if (daysToKeep2 > 0 && daysToKeep2 != pDb->cfg.daysToKeep2) { + mTrace("db:%s, daysToKeep2:%d change to %d", pDb->name, pDb->cfg.daysToKeep2, daysToKeep2); + newCfg.daysToKeep2 = daysToKeep2; + } + + if (compression > 0 && compression != pDb->cfg.compression) { + mTrace("db:%s, compression:%d change to %d", pDb->name, pDb->cfg.compression, compression); + newCfg.compression = compression; + } + + if (replications > 0 && replications != pDb->cfg.replications) { + mTrace("db:%s, replications:%d change to %d", pDb->name, pDb->cfg.replications, replications); + newCfg.replications = replications; + } + if (replications > mgmtGetDnodesNum()) { + mError("db:%s, no enough dnode to change replica:%d", pDb->name, replications); + terrno = TSDB_CODE_NO_ENOUGH_DNODES; + } + return newCfg; } @@ -757,8 +784,16 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { return terrno; } + int32_t code = mgmtCheckDbCfg(&newCfg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t oldReplica = pDb->cfg.replications; + if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; + pDb->cfgVersion++; SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsDbSdb, @@ -771,7 +806,20 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) { return TSDB_CODE_SDB_ERROR; } } - + + void *pNode = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pNode = mgmtGetNextVgroup(pNode, &pVgroup); + if (pVgroup == NULL) break; + mgmtSendCreateVgroupMsg(pVgroup, NULL); + mgmtDecVgroupRef(pVgroup); + } + + if (oldReplica != pDb->cfg.replications) { + balanceNotify(); + } + return TSDB_CODE_SUCCESS; } @@ -799,16 +847,6 @@ static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) { return; } - SVgObj *pVgroup = pDb->pHead; - if (pVgroup != NULL) { - mPrint("vgroup:%d, will be altered", pVgroup->vgId); - SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg); - newMsg->ahandle = pVgroup; - newMsg->expected = pVgroup->numOfVnodes; - mgmtAlterVgroup(pVgroup, newMsg); - return; - } - mTrace("db:%s, all vgroups is altered", pDb->name); mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS); } diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index 8afeadeb0c..efa0b03ded 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -336,13 +336,14 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { if (pStatus->dnodeId == 0) { mTrace("dnode:%d, first access, privateIp:%s, name:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName); } else { - //mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); + mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); } int32_t openVnodes = htons(pStatus->openVnodes); for (int32_t j = 0; j < openVnodes; ++j) { SVnodeLoad *pVload = &pStatus->load[j]; pVload->vgId = htonl(pVload->vgId); + pVload->cfgVersion = htonl(pVload->cfgVersion); SVgObj *pVgroup = mgmtGetVgroup(pVload->vgId); if (pVgroup == NULL) { diff --git a/src/mnode/src/mgmtVgroup.c b/src/mnode/src/mgmtVgroup.c index feedf5fdc3..45512b86b3 100644 --- a/src/mnode/src/mgmtVgroup.c +++ b/src/mnode/src/mgmtVgroup.c @@ -272,8 +272,9 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo pVgroup->pointsWritten = htobe64(pVload->pointsWritten); } - if (pVload->replica != pVgroup->numOfVnodes) { - mError("dnode:%d, vgroup:%d replica:%d not match with mgmt:%d", pDnode->dnodeId, pVload->vgId, pVload->replica, + if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) { + mError("dnode:%d, vgroup:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d", + pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion, pVgroup->numOfVnodes); mgmtSendCreateVgroupMsg(pVgroup, NULL); } @@ -535,6 +536,7 @@ SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) { SMDVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); + pCfg->cfgVersion = htonl(pDb->cfgVersion); pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks); pCfg->maxTables = htonl(pDb->cfg.maxTables); @@ -769,15 +771,3 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) { mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups); } - -void mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle) { - assert(ahandle != NULL); - - if (pVgroup->numOfVnodes != pVgroup->pDb->cfg.replications) { - // TODO: - // mgmtSendAlterVgroupMsg(pVgroup, NULL); - } else { - mgmtAddToShellQueue(ahandle); - } -} - diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index f1b9d55d54..d99cb1c96c 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -36,6 +36,7 @@ typedef struct { void *sync; void *events; void *cq; // continuous query + int32_t cfgVersion; STsdbCfg tsdbCfg; SSyncCfg syncCfg; SWalCfg walCfg; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 153bdbffd8..96470c4e3a 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -331,6 +331,7 @@ static void vnodeBuildVloadMsg(char *pNode, void * param) { SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; pLoad->vgId = htonl(pVnode->vgId); + pLoad->cfgVersion = htonl(pVnode->cfgVersion); pLoad->totalStorage = htobe64(pLoad->totalStorage); pLoad->compStorage = htobe64(pLoad->compStorage); pLoad->pointsWritten = htobe64(pLoad->pointsWritten); @@ -389,6 +390,7 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { len += snprintf(content + len, maxLen - len, "{\n"); + len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion); len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); @@ -463,6 +465,13 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_OVER; } + cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); + if (!cfgVersion || cfgVersion->type != cJSON_Number) { + dError("pVnode:%p vgId:%d, failed to read vnode cfg, cfgVersion not found", pVnode, pVnode->vgId); + goto PARSE_OVER; + } + pVnode->cfgVersion = cfgVersion->valueint; + cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { dError("pVnode:%p vgId:%d, failed to read vnode cfg, cacheBlockSize not found", pVnode, pVnode->vgId); From 238879bc80e7a934a267c41e4656eb794fc65aa9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 26 Apr 2020 15:34:15 +0800 Subject: [PATCH 60/63] add None type support in sql.py. --- tests/pytest/simpletest.sh | 22 +++++++++++++++++++--- tests/pytest/util/sql.py | 7 ++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh index 58345e13ee..73b25d2056 100755 --- a/tests/pytest/simpletest.sh +++ b/tests/pytest/simpletest.sh @@ -1,9 +1,25 @@ #!/bin/bash -python3 ./test.py -f insert/basic.py $1 +python3 ./test.py $1 -f insert/basic.py python3 ./test.py -s $1 sleep 1 -python3 ./test.py -f insert/int.py $1 +python3 ./test.py $1 -f insert/int.py python3 ./test.py -s $1 sleep 1 -python3 ./test.py -f insert/float.py $1 +python3 ./test.py $1 -f insert/float.py python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/bigint.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/bool.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/double.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/smallint.py +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py $1 -f insert/tinyint.py +python3 ./test.py -s $1 +sleep 1 diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 0008468130..edfee4ddb1 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -89,7 +89,12 @@ class TDSql: tdLog.exit( "sql:%.40s row:%d col:%d data:%s != expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) - tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" % + + if data is None: + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + else: + tdLog.info("sql:%.40s, row:%d col:%d data:%s == expect:%d" % (self.sql, row, col, self.queryResult[row][col], data)) def getData(self, row, col): From d1624536c52ed81aa0aaa01049f099a64ae10417 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 26 Apr 2020 16:20:34 +0800 Subject: [PATCH 61/63] remove log --- src/mnode/src/mgmtDnode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mgmtDnode.c b/src/mnode/src/mgmtDnode.c index efa0b03ded..5d34d3fcf2 100644 --- a/src/mnode/src/mgmtDnode.c +++ b/src/mnode/src/mgmtDnode.c @@ -336,7 +336,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) { if (pStatus->dnodeId == 0) { mTrace("dnode:%d, first access, privateIp:%s, name:%s", pDnode->dnodeId, taosIpStr(pDnode->privateIp), pDnode->dnodeName); } else { - mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); + //mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); } int32_t openVnodes = htons(pStatus->openVnodes); From 2de6d7387151d682af820310dc1f684014024277 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 26 Apr 2020 08:49:27 +0000 Subject: [PATCH 62/63] gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e91d6739a5..d0af51d31b 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,5 @@ Target/ *.failed *.sql sim/ +*DS_Store From d115fe573e131ea2b46ac0300bbf23baa75cf769 Mon Sep 17 00:00:00 2001 From: slguan Date: Sun, 26 Apr 2020 19:02:59 +0800 Subject: [PATCH 63/63] [TD-19] --- src/mnode/src/mgmtDb.c | 2 +- src/plugins/http/src/httpResp.c | 2 +- src/plugins/http/src/httpSql.c | 2 +- tests/script/general/http/restful_full.sim | 20 ++++++++++---------- tests/script/general/http/telegraf.sim | 4 ++-- tests/script/general/http/testSuite.sim | 6 +++--- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 29d2343231..c7f6ef4a4d 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -233,7 +233,7 @@ static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) { return TSDB_CODE_INVALID_OPTION; } - if (pCfg->precision != TSDB_MIN_PRECISION && pCfg->precision != TSDB_MAX_PRECISION) { + if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) { mError("invalid db option timePrecision:%d valid value: [%d, %d]", pCfg->precision, TSDB_MIN_PRECISION, TSDB_MAX_PRECISION); return TSDB_CODE_INVALID_OPTION; diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 0e2e7b3f85..d9507072de 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -199,7 +199,7 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) { } else {} } - httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_INVALID_SQL, temp); + httpSendErrorRespImp(pContext, httpCode, "Bad Request", 1000, temp); } void httpSendSuccResp(HttpContext *pContext, char *desc) { diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 0d0bc3e0fd..45cc3fcfdf 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -239,7 +239,7 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) { if (code < 0) { SSqlObj *pObj = (SSqlObj *)result; if (code == TSDB_CODE_INVALID_SQL) { - httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s:invalidsql, sqlObj:%p, error:%s", + httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload); httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload); } else { diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 6682aac352..b42f440068 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -83,19 +83,19 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 192.168.0.1:6020/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created time","ntables","vgroups","replica","days","keep1,keep2,keep(D)","tables","rows","cache(b)","ablocks","tblocks","ctime(s)","clog","comp","time precision","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","create time","ntables","vgroups","replica","days","keep1,keep2,keep(D)","tables","cache(MB)","blocks","minrows","maxrows","ctime(s)","clog","comp","precision","status"],"data":[],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 192.168.0.1:6020/rest/sql print 12-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 192.168.0.1:6020/rest/sql print 13-> $system_content -if $system_content != @{"status":"error","code":33,"desc":"DB already there"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"DB already there"}@ then return -1 endi @@ -121,14 +121,14 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' used1' 192.168.0.1:6020/rest/sql print 17-> $system_content -if $system_content != @{"status":"error","code":28,"desc":"invalid SQL: invalid SQL: syntax error near 'used1'"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"invalid SQL: invalid SQL: syntax error near 'used1'"}@ then return -1 endi #18 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 192.168.0.1:6020/rest/sql print 18-> $system_content -if $system_content != @{"status":"error","code":38,"desc":"DB not selected"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"db not selected"}@ then return -1 endi @@ -137,7 +137,7 @@ print =============== step3 - db #19 #system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' show tables;' 192.168.0.1:6020/rest/sql/d4 #print 19-> $system_content -#if $system_content != @{"status":"error","code":102,"desc":"invalid DB"}@ then +#if $system_content != @{"status":"error","code":1000,"desc":"invalid DB"}@ then # return -1 #endi @@ -149,13 +149,13 @@ print =============== step3 - db system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1;' 192.168.0.1:6020/rest/sql print 21-> $system_content -if $system_content != @{"status":"error","code":32,"desc":"invalid table"}@ then +if $system_content != @{"status":"error","code":1000,"desc":"invalid table name"}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d1.t1 (ts timestamp, speed int)' 192.168.0.1:6020/rest/sql print 22-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then return -1 endi @@ -209,13 +209,13 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d2' 192.168.0.1:6020/rest/sql print 28-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d2.t1 (ts timestamp, speed int)' 192.168.0.1:6020/rest/sql print 29-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then return -1 endi diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim index 1667034f0e..b782a9b9bf 100644 --- a/tests/script/general/http/telegraf.sim +++ b/tests/script/general/http/telegraf.sim @@ -106,7 +106,7 @@ if $system_content != @{"status":"error","code":1029,"desc":"metric name length return -1 endi -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 +system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234a1234567890123456789012345678901234","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content if $system_content != @{"status":"error","code":1030,"desc":"metric name length too long"}@ then @@ -204,7 +204,7 @@ if $system_content != @{"status":"error","code":1042,"desc":"table is null"}@ th return -1 endi -system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"host111111111111222222222222222222222host111111111111222222222222222222222"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 +system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222host111111111111222222222222222222222"},"timestamp":1535784122}' 127.0.0.1:6020/telegraf/db/root/taosdata1 print $system_content if $system_content != @{"status":"error","code":1043,"desc":"table name length too long"}@ then diff --git a/tests/script/general/http/testSuite.sim b/tests/script/general/http/testSuite.sim index cec8da67bb..336d606a37 100644 --- a/tests/script/general/http/testSuite.sim +++ b/tests/script/general/http/testSuite.sim @@ -3,6 +3,6 @@ run general/http/restful_insert.sim #run general/http/restful_limit.sim #run general/http/restful_full.sim #run general/http/prepare.sim -run general/http/telegraf.sim -run general/http/grafana_bug.sim -run general/http/grafana.sim +#run general/http/telegraf.sim +#run general/http/grafana_bug.sim +#run general/http/grafana.sim