From ca985ec7c61448967aa03d0b1409fbf0c5007983 Mon Sep 17 00:00:00 2001 From: wpan Date: Fri, 30 Jul 2021 10:38:15 +0800 Subject: [PATCH 1/5] fix interp bug --- src/query/inc/qExecutor.h | 3 + src/query/src/qAggMain.c | 52 +- src/query/src/qExecutor.c | 373 +++++- tests/script/general/parser/interp.sim | 5 +- tests/script/general/parser/interp_test.sim | 1321 ++++++++++++++++++- 5 files changed, 1725 insertions(+), 29 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 31ac70d6cd..4d961e5332 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -183,6 +183,7 @@ typedef struct SQuery { bool stableQuery; // super table query or not bool topBotQuery; // TODO used bitwise flag + bool interpQuery; // denote if this is an interp query bool groupbyColumn; // denote if this is a groupby normal column query bool hasTagResults; // if there are tag values in final result or not bool timeWindowInterpo;// if the time window start/end required interpolation @@ -288,6 +289,8 @@ enum OPERATOR_TYPE_E { OP_MultiTableAggregate = 14, OP_MultiTableTimeInterval = 15, OP_Having = 16, + OP_AllTimeWindow = 17, + OP_AllMultiTableTimeInterval = 18, }; typedef struct SOperatorInfo { diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index f312b4ab64..b3996fb55a 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4278,27 +4278,59 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { } } else { // no data generated yet - if (pCtx->size == 1) { + if (pCtx->size < 1) { return; } // check the timestamp in input buffer TSKEY skey = GET_TS_DATA(pCtx, 0); - TSKEY ekey = GET_TS_DATA(pCtx, 1); - - // no data generated yet - if (!(skey < pCtx->startTs && ekey > pCtx->startTs)) { - return; - } - - assert(pCtx->start.key == INT64_MIN && skey < pCtx->startTs && ekey > pCtx->startTs); if (type == TSDB_FILL_PREV) { + if (skey > pCtx->startTs) { + return; + } + + if (pCtx->size > 1) { + TSKEY ekey = GET_TS_DATA(pCtx, 1); + if (ekey > skey && ekey <= pCtx->startTs) { + skey = ekey; + } + } assignVal(pCtx->pOutput, pCtx->pInput, pCtx->outputBytes, pCtx->inputType); } else if (type == TSDB_FILL_NEXT) { - char* val = ((char*)pCtx->pInput) + pCtx->inputBytes; + TSKEY ekey = skey; + char* val = NULL; + + if (ekey < pCtx->startTs) { + if (pCtx->size > 1) { + ekey = GET_TS_DATA(pCtx, 1); + if (ekey < pCtx->startTs) { + return; + } + + val = ((char*)pCtx->pInput) + pCtx->inputBytes; + } else { + return; + } + } else { + val = (char*)pCtx->pInput; + } + assignVal(pCtx->pOutput, val, pCtx->outputBytes, pCtx->inputType); } else if (type == TSDB_FILL_LINEAR) { + if (pCtx->size <= 1) { + return; + } + + TSKEY ekey = GET_TS_DATA(pCtx, 1); + + // no data generated yet + if (!(skey < pCtx->startTs && ekey > pCtx->startTs)) { + return; + } + + assert(pCtx->start.key == INT64_MIN && skey < pCtx->startTs && ekey > pCtx->startTs); + char *start = GET_INPUT_DATA(pCtx, 0); char *end = GET_INPUT_DATA(pCtx, 1); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9b5b23d9aa..9d516e976f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -200,11 +200,13 @@ static SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOp static SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); static SOperatorInfo* createOffsetOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); static SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); static SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); static SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); static SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); static SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); static SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); static SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); static SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); static SOperatorInfo* createHavingOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); @@ -448,6 +450,37 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntim pResultRowInfo->capacity = (int32_t)newCapacity; } +static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, + int16_t bytes, bool masterscan, uint64_t uid) { + bool existed = false; + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); + + SResultRow **p1 = + (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return p1 != NULL; + } + + if (p1 != NULL) { + for(int32_t i = pResultRowInfo->size - 1; i >= 0; --i) { + if (pResultRowInfo->pResult[i] == (*p1)) { + pResultRowInfo->curIndex = i; + existed = true; + break; + } + } + } + + return existed; + } + + return p1 != NULL; +} + + static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { bool existed = false; @@ -586,6 +619,42 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t return w; } +// get the correct time window according to the handled timestamp +static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQuery *pQuery) { + STimeWindow w = {0}; + + if (pResultRowInfo->curIndex == -1) { // the first window, from the previous stored value + if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { + getInitialStartTimeWindow(pQuery, ts, &w); + pResultRowInfo->prevSKey = w.skey; + } else { + w.skey = pResultRowInfo->prevSKey; + } + + if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { + w.ekey = taosTimeAdd(w.skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1; + } else { + w.ekey = w.skey + pQuery->interval.interval - 1; + } + } else { + int32_t slot = curTimeWindowIndex(pResultRowInfo); + SResultRow* pWindowRes = getResultRow(pResultRowInfo, slot); + w = pWindowRes->win; + } + + /* + * query border check, skey should not be bounded by the query time range, since the value skey will + * be used as the time window index value. So we only change ekey of time window accordingly. + */ + if (w.ekey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) { + w.ekey = pQuery->window.ekey; + } + + return w; +} + + + // a new buffer page for each table. Needs to opt this design static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, uint32_t size) { if (pWindowRes->pageId != -1) { @@ -631,6 +700,17 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } + +static bool chkWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, + bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { + assert(win->skey <= win->ekey); + + return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); +} + + + static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { @@ -701,7 +781,7 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se } } - assert(forwardStep > 0); + assert(forwardStep >= 0); return forwardStep; } @@ -751,9 +831,9 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } else { pResultRowInfo->curIndex = i + 1; // current not closed result object } - - pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; } + + pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; } static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQuery* pQuery, TSKEY lastKey) { @@ -803,7 +883,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo } } - assert(num > 0); + assert(num >= 0); return num; } @@ -885,6 +965,11 @@ static int32_t getNextQualifiedWindow(SQuery* pQuery, STimeWindow *pNext, SDataB } } + /* interp query with fill should not skip time window */ + if (pQuery->interpQuery && pQuery->fillType != TSDB_FILL_NONE) { + return startPos; + } + /* * This time window does not cover any data, try next time window, * this case may happen when the time window is too small @@ -1351,6 +1436,81 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul updateResultRowInfoActiveIndex(pResultRowInfo, pQuery, pQuery->current->lastKey); } + +static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t groupId) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + SQuery* pQuery = pRuntimeEnv->pQuery; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); + tsCols = (int64_t*) pColDataInfo->pData; + assert(tsCols[0] == pSDataBlock->info.window.skey && + tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); + } + + int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(pQuery, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + + STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQuery); + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + + SResultRow* pResult = NULL; + int32_t forwardStep = 0; + + while (1) { + // null data, failed to allocate more memory buffer + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &win, masterScan, &pResult, groupId, + pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + TSKEY ekey = reviseWindowEkey(pQuery, &win); + forwardStep = getNumOfRowsInTimeWindow(pQuery, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(pQuery, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); + if (startPos < 0) { + if (win.skey <= pQuery->window.ekey) { + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &win, masterScan, &pResult, groupId, + pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + startPos = pSDataBlock->info.rows - 1; + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + } + + break; + } + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + + if (pQuery->timeWindowInterpo) { + int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; + saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); + } + + updateResultRowInfoActiveIndex(pResultRowInfo, pQuery, pQuery->current->lastKey); +} + + + static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; STableQueryInfo* item = pRuntimeEnv->pQuery->current; @@ -1788,12 +1948,24 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf } else if (QUERY_IS_INTERVAL_QUERY(pQuery)) { if (pQuery->stableQuery) { - pRuntimeEnv->proot = createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, - pQuery->pExpr1, pQuery->numOfOutput); + if (pQuery->interpQuery) { + pRuntimeEnv->proot = createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, + pQuery->pExpr1, pQuery->numOfOutput); + } else { + pRuntimeEnv->proot = createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, + pQuery->pExpr1, pQuery->numOfOutput); + } setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); } else { - pRuntimeEnv->proot = - createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + if (pQuery->interpQuery) { + pRuntimeEnv->proot = + createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + + } else { + pRuntimeEnv->proot = + createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + } + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); if (pQuery->pExpr2 != NULL) { @@ -1801,12 +1973,11 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQuery->pExpr2, pQuery->numOfExpr2); } - if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { + if (pQuery->fillType != TSDB_FILL_NONE) { SOperatorInfo* pInfo = pRuntimeEnv->proot; pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput); } } - } else if (pQuery->groupbyColumn) { pRuntimeEnv->proot = createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); @@ -2666,6 +2837,8 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa // check if this data block is required to load if ((*status) != BLK_DATA_ALL_NEEDED) { + bool needFilter = true; + // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer if (QUERY_IS_INTERVAL_QUERY(pQuery)) { @@ -2675,18 +2848,20 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + pTableScanInfo->rowCellInfoOffset); } else if (pQuery->stableQuery && (!isTsCompQuery(pQuery))) { // stable aggregate, not interval aggregate or normal column aggregate doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->rowCellInfoOffset, pTableScanInfo->numOfOutput, pQuery->current->groupIndex); } - (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); + if (needFilter) { + (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); + } else { + (*status) = BLK_DATA_ALL_NEEDED; + } } SDataBlockInfo* pBlockInfo = &pBlock->info; @@ -4104,6 +4279,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pQuery->tsdb = tsdb; pQuery->topBotQuery = isTopBottomQuery(pQuery); + pQuery->interpQuery = isPointInterpoQuery(pQuery); pQuery->hasTagResults = hasTagValOutput(pQuery); pQuery->timeWindowInterpo = timeWindowInterpoRequired(pQuery); pQuery->stabledev = isStabledev(pQuery); @@ -4471,7 +4647,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf pTableScanInfo->pCtx = pAggInfo->binfo.pCtx; pTableScanInfo->pResultRowInfo = &pAggInfo->binfo.resultRowInfo; pTableScanInfo->rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_TimeWindow) { + } else if (pDownstream->operatorType == OP_TimeWindow || pDownstream->operatorType == OP_AllTimeWindow) { STableIntervalOperatorInfo *pIntervalInfo = pDownstream->info; pTableScanInfo->pCtx = pIntervalInfo->pCtx; @@ -4485,7 +4661,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf pTableScanInfo->pResultRowInfo = &pGroupbyInfo->binfo.resultRowInfo; pTableScanInfo->rowCellInfoOffset = pGroupbyInfo->binfo.rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_MultiTableTimeInterval) { + } else if (pDownstream->operatorType == OP_MultiTableTimeInterval || pDownstream->operatorType == OP_AllMultiTableTimeInterval) { STableIntervalOperatorInfo *pInfo = pDownstream->info; pTableScanInfo->pCtx = pInfo->pCtx; @@ -4911,6 +5087,64 @@ static SSDataBlock* doIntervalAgg(void* param) { return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; } +static SSDataBlock* doAllIntervalAgg(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes; + } + + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; + STimeWindow win = pQuery->window; + + SOperatorInfo* upstream = pOperator->upstream; + + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { + break; + } + + setTagValue(pOperator, pRuntimeEnv->pQuery->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQuery->order.order); + hashAllIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + } + + // restore the value + pQuery->order.order = order; + pQuery->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pIntervalInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; +} + + static SSDataBlock* doSTableIntervalAgg(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { @@ -4963,6 +5197,60 @@ static SSDataBlock* doSTableIntervalAgg(void* param) { return pIntervalInfo->pRes; } +static SSDataBlock* doAllSTableIntervalAgg(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + if (pOperator->status == OP_RES_TO_RETURN) { + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes; + } + + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; + + SOperatorInfo* upstream = pOperator->upstream; + + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->pQuery->current; + + setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQuery->order.order); + setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); + + hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); + } + + pOperator->status = OP_RES_TO_RETURN; + pQuery->order.order = order; // TODO : restore the order + doCloseAllTimeWindow(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes; +} + + + static SSDataBlock* doSessionWindowAgg(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { @@ -5406,6 +5694,32 @@ SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOp return pOperator; } +SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "AllTimeIntervalAggOperator"; + pOperator->operatorType = OP_AllTimeWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doAllIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; +} + + + SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); @@ -5455,6 +5769,31 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRunti return pOperator; } +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "AllMultiTableTimeIntervalOperator"; + pOperator->operatorType = OP_AllMultiTableTimeInterval; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doAllSTableIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; +} + + SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); pInfo->colIndex = -1; // group by column index diff --git a/tests/script/general/parser/interp.sim b/tests/script/general/parser/interp.sim index 3fb91e36c6..55c5701985 100644 --- a/tests/script/general/parser/interp.sim +++ b/tests/script/general/parser/interp.sim @@ -55,6 +55,9 @@ while $i < $halfNum endw print ====== tables created +sql create table ap1 (ts timestamp, pav float); +sql INSERT INTO ap1 VALUES ('2021-07-25 02:19:54.100',1) ('2021-07-25 02:19:54.200',2) ('2021-07-25 02:19:54.300',3) ('2021-07-25 02:19:56.500',4) ('2021-07-25 02:19:57.500',5) ('2021-07-25 02:19:57.600',6) ('2021-07-25 02:19:57.900',7) ('2021-07-25 02:19:58.100',8) ('2021-07-25 02:19:58.300',9) ('2021-07-25 02:19:59.100',10) ('2021-07-25 02:19:59.300',11) ('2021-07-25 02:19:59.500',12) ('2021-07-25 02:19:59.700',13) ('2021-07-25 02:19:59.900',14) ('2021-07-25 02:20:05.000', 20) ('2021-07-25 02:25:00.000', 10000); + run general/parser/interp_test.sim print ================== restart server to commit data into disk @@ -65,4 +68,4 @@ print ================== server restart completed run general/parser/interp_test.sim -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/interp_test.sim b/tests/script/general/parser/interp_test.sim index 81a77995fb..845afb0173 100644 --- a/tests/script/general/parser/interp_test.sim +++ b/tests/script/general/parser/interp_test.sim @@ -927,4 +927,1323 @@ endi if $data44 != @18-11-25 19:06:00.000@ then return -1 -endi \ No newline at end of file +endi + + + + + + + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:20:00' interval(1s) fill(linear); +if $rows != 6 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.31818 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.77273 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.50000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.50000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.87500 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:20:00' interval(1s) fill(value, 1); +if $rows != 6 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 1.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 1.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 1.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 1.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 1.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:20:00' interval(1s) fill(NULL); +if $rows != 6 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != NULL then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != NULL then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != NULL then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != NULL then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:20:00' interval(1s) fill(prev); +if $rows != 6 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:20:00' interval(1s) fill(next); +if $rows != 6 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 4.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 4.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 5.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 8.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 10.00000 then + return -1 +endi + + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:19:56' interval(1s) fill(linear); +if $rows != 0 then + return -1 +endi +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:19:56' interval(1s) fill(prev); +if $rows != 2 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:19:56' interval(1s) fill(next); +if $rows != 2 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != NULL then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:19:57' interval(1s) fill(linear); +if $rows != 3 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.31818 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.77273 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:19:57' interval(1s) fill(prev); +if $rows != 3 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:19:57' interval(1s) fill(next); +if $rows != 3 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 4.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 4.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:20:03' interval(1s) fill(linear); +if $rows != 10 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.31818 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.77273 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.50000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.50000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.87500 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != NULL then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != NULL then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != NULL then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != NULL then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:20:03' interval(1s) fill(prev); +if $rows != 10 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 14.00000 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 14.00000 then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != 14.00000 then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != 14.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:20:03' interval(1s) fill(next); +if $rows != 10 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 4.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 4.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 5.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 8.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 10.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != NULL then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != NULL then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != NULL then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != NULL then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:20:05' interval(1s) fill(linear); +if $rows != 12 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.31818 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.77273 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.50000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.50000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.87500 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 14.11765 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 15.29412 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:20:05' interval(1s) fill(prev); +if $rows != 12 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 14.00000 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 14.00000 then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != 14.00000 then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != 14.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:20:05' interval(1s) fill(next); +if $rows != 12 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 4.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 4.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 5.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 8.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 10.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 20.00000 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 20.00000 then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != 20.00000 then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != 20.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:20:02' and ts<='2021-07-25 02:20:05' interval(1s) fill(value, 1); +if $rows != 4 then + return -1 +endi +if $data00 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data11 != 1.00000 then + return -1 +endi +if $data20 != @21-07-25 02:20:04.000@ then + return -1 +endi +if $data21 != 1.00000 then + return -1 +endi +if $data30 != @21-07-25 02:20:05.000@ then + return -1 +endi +if $data31 != 20.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:20:02' and ts<='2021-07-25 02:20:05' interval(1s) fill(null); +if $rows != 4 then + return -1 +endi +if $data00 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data11 != NULL then + return -1 +endi +if $data20 != @21-07-25 02:20:04.000@ then + return -1 +endi +if $data21 != NULL then + return -1 +endi +if $data30 != @21-07-25 02:20:05.000@ then + return -1 +endi +if $data31 != 20.00000 then + return -1 +endi + + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:20:25' interval(1s) fill(linear); +if $rows != 32 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.31818 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.77273 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.50000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.50000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.87500 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 14.11765 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 15.29412 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:20:25' interval(1s) fill(prev); +if $rows != 32 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 14.00000 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 14.00000 then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != 14.00000 then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != 14.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:20:25' interval(1s) fill(next); +if $rows != 32 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 4.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 4.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 5.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 8.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 10.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 20.00000 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 20.00000 then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != 20.00000 then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != 20.00000 then + return -1 +endi + + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:25:00' interval(1s) fill(linear); +if $rows != 307 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.31818 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.77273 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.50000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.50000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.87500 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 14.11765 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 15.29412 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:25:00' interval(1s) fill(prev); +if $rows != 307 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 14.00000 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 14.00000 then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != 14.00000 then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != 14.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 02:25:00' interval(1s) fill(next); +if $rows != 307 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 4.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 4.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 5.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 8.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 10.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 20.00000 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 20.00000 then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != 20.00000 then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != 20.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 03:25:00' interval(1s) fill(linear); +if $rows != 3907 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.31818 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.77273 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.50000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.50000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.87500 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 14.11765 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 15.29412 then + return -1 +endi + + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 03:25:00' interval(1s) fill(prev); +if $rows != 3907 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 3.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 4.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 7.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 9.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 14.00000 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 14.00000 then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != 14.00000 then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != 14.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<='2021-07-25 03:25:00' interval(1s) fill(next); +if $rows != 3907 then + return -1 +endi +if $data00 != @21-07-25 02:19:54.000@ then + return -1 +endi +if $data01 != 1.00000 then + return -1 +endi +if $data10 != @21-07-25 02:19:55.000@ then + return -1 +endi +if $data11 != 4.00000 then + return -1 +endi +if $data20 != @21-07-25 02:19:56.000@ then + return -1 +endi +if $data21 != 4.00000 then + return -1 +endi +if $data30 != @21-07-25 02:19:57.000@ then + return -1 +endi +if $data31 != 5.00000 then + return -1 +endi +if $data40 != @21-07-25 02:19:58.000@ then + return -1 +endi +if $data41 != 8.00000 then + return -1 +endi +if $data50 != @21-07-25 02:19:59.000@ then + return -1 +endi +if $data51 != 10.00000 then + return -1 +endi +if $data60 != @21-07-25 02:20:00.000@ then + return -1 +endi +if $data61 != 20.00000 then + return -1 +endi +if $data70 != @21-07-25 02:20:01.000@ then + return -1 +endi +if $data71 != 20.00000 then + return -1 +endi +if $data80 != @21-07-25 02:20:02.000@ then + return -1 +endi +if $data81 != 20.00000 then + return -1 +endi +if $data90 != @21-07-25 02:20:03.000@ then + return -1 +endi +if $data91 != 20.00000 then + return -1 +endi + +sql select interp(pav) from ap1 where ts> '2021-07-25 02:19:54' and ts<'2021-07-25 02:20:07' interval(1s); +if $rows != 1 then + return -1 +endi +if $data00 != @21-07-25 02:20:05.000@ then + return -1 +endi +if $data01 != 20.00000 then + return -1 +endi + From 3102cdee2e8462346339a7bfca8620e0180f4e54 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 30 Jul 2021 14:18:05 +0800 Subject: [PATCH 2/5] Hotfix/sangshuduo/td 5445 for master (#7083) * [TD-5445]: taosdemo bug stmt interface with sample data. sync from develop branch. * refactoring some macros. * fix mac compile error. --- src/kit/taosdemo/taosdemo.c | 133 +++++++++++++++++++++--------------- 1 file changed, 77 insertions(+), 56 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 8a4b781d3f..a7a4567b97 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -66,10 +66,6 @@ extern char configDir[]; -#define INSERT_JSON_NAME "insert.json" -#define QUERY_JSON_NAME "query.json" -#define SUBSCRIBE_JSON_NAME "subscribe.json" - #define STR_INSERT_INTO "INSERT INTO " #define MAX_RECORDS_PER_REQ 32766 @@ -79,9 +75,10 @@ extern char configDir[]; #define BUFFER_SIZE TSDB_MAX_ALLOWED_SQL_LEN #define COND_BUF_LEN (BUFFER_SIZE - 30) #define COL_BUFFER_LEN ((TSDB_COL_NAME_LEN + 15) * TSDB_MAX_COLUMNS) + #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 -#define MAX_HOSTNAME_SIZE 64 +#define MAX_HOSTNAME_SIZE 253 // https://man7.org/linux/man-pages/man7/hostname.7.html #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ @@ -99,6 +96,11 @@ extern char configDir[]; #define MAX_DATABASE_COUNT 256 #define INPUT_BUF_LEN 256 +#define TBNAME_PREFIX_LEN (TSDB_TABLE_NAME_LEN - 20) // 20 characters reserved for seq +#define SMALL_BUFF_LEN 8 +#define DATATYPE_BUFF_LEN (SMALL_BUFF_LEN*3) +#define NOTE_BUFF_LEN (SMALL_BUFF_LEN*16) + #define DEFAULT_TIMESTAMP_STEP 1 @@ -243,16 +245,15 @@ typedef struct SArguments_S { typedef struct SColumn_S { char field[TSDB_COL_NAME_LEN]; - char dataType[16]; + char dataType[DATATYPE_BUFF_LEN]; uint32_t dataLen; - char note[128]; + char note[NOTE_BUFF_LEN]; } StrColumn; typedef struct SSuperTable_S { char sTblName[TSDB_TABLE_NAME_LEN]; - char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample - char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq - char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest + char dataSource[SMALL_BUFF_LEN]; // rand_gen or sample + char childTblPrefix[TBNAME_PREFIX_LEN]; uint16_t childTblExists; int64_t childTblCount; uint64_t batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql @@ -271,7 +272,7 @@ typedef struct SSuperTable_S { int64_t insertRows; int64_t timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; - char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json + char sampleFormat[SMALL_BUFF_LEN]; // csv, json char sampleFile[MAX_FILE_NAME_LEN]; char tagsFile[MAX_FILE_NAME_LEN]; @@ -307,7 +308,7 @@ typedef struct { int16_t replica; int16_t quorum; int16_t days; - char keeplist[32]; + char keeplist[64]; int32_t cache; //MB int32_t blocks; int32_t minrows; @@ -316,7 +317,7 @@ typedef struct { int32_t fsync; int8_t comp; int8_t cachelast; - char precision[8]; // time resolution + char precision[SMALL_BUFF_LEN]; // time resolution int8_t update; char status[16]; } SDbInfo; @@ -336,7 +337,7 @@ typedef struct SDbCfg_S { int cache; int blocks; int quorum; - char precision[8]; + char precision[SMALL_BUFF_LEN]; } SDbCfg; typedef struct SDataBase_S { @@ -402,7 +403,7 @@ typedef struct SuperQueryInfo_S { int subscribeKeepProgress; uint64_t queryTimes; int64_t childTblCount; - char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq + char childTblPrefix[TBNAME_PREFIX_LEN]; // 20 characters reserved for seq int sqlCount; char sql[MAX_QUERY_SQL_COUNT][BUFFER_SIZE+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; @@ -422,7 +423,7 @@ typedef struct SQueryMetaInfo_S { char user[MAX_USERNAME_SIZE]; char password[MAX_PASSWORD_SIZE]; char dbName[TSDB_DB_NAME_LEN]; - char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest + char queryMode[SMALL_BUFF_LEN]; // taosc, rest SpecifiedQueryInfo specifiedQueryInfo; SuperQueryInfo superQueryInfo; @@ -2679,12 +2680,14 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); tstrncpy(superTbls->tags[tagIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); superTbls->tags[tagIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); tstrncpy(superTbls->tags[tagIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + min(NOTE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes) + 1); tagIndex++; } else { tstrncpy(superTbls->columns[columnIndex].field, @@ -2692,12 +2695,14 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); tstrncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); superTbls->columns[columnIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); tstrncpy(superTbls->columns[columnIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + min(NOTE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes) + 1); columnIndex++; } count++; @@ -3460,8 +3465,9 @@ static bool getColumnAndTagTypeFromInsertJsonFile( __func__, __LINE__); goto PARSE_OVER; } - //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); - tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); + //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, DATATYPE_BUFF_LEN); + tstrncpy(columnCase.dataType, dataType->valuestring, + min(DATATYPE_BUFF_LEN, strlen(dataType->valuestring) + 1)); cJSON* dataLen = cJSON_GetObjectItem(column, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3471,12 +3477,13 @@ static bool getColumnAndTagTypeFromInsertJsonFile( __func__, __LINE__); goto PARSE_OVER; } else { - columnCase.dataLen = 8; + columnCase.dataLen = SMALL_BUFF_LEN; } for (int n = 0; n < count; ++n) { tstrncpy(superTbls->columns[index].dataType, - columnCase.dataType, strlen(columnCase.dataType) + 1); + columnCase.dataType, + min(DATATYPE_BUFF_LEN, strlen(columnCase.dataType) + 1)); superTbls->columns[index].dataLen = columnCase.dataLen; index++; } @@ -3532,7 +3539,8 @@ static bool getColumnAndTagTypeFromInsertJsonFile( __func__, __LINE__); goto PARSE_OVER; } - tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); + tstrncpy(columnCase.dataType, dataType->valuestring, + min(DATATYPE_BUFF_LEN, strlen(dataType->valuestring) + 1)); cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3547,7 +3555,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( for (int n = 0; n < count; ++n) { tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, - strlen(columnCase.dataType) + 1); + min(DATATYPE_BUFF_LEN, strlen(columnCase.dataType) + 1)); superTbls->tags[index].dataLen = columnCase.dataLen; index++; } @@ -3791,9 +3799,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (precision && precision->type == cJSON_String && precision->valuestring != NULL) { tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, - 8); + SMALL_BUFF_LEN); } else if (!precision) { - memset(g_Dbs.db[i].dbCfg.precision, 0, 8); + memset(g_Dbs.db[i].dbCfg.precision, 0, SMALL_BUFF_LEN); } else { printf("ERROR: failed to read json, precision not found\n"); goto PARSE_OVER; @@ -3978,7 +3986,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, - TSDB_TABLE_NAME_LEN - 20); + TBNAME_PREFIX_LEN); cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); if (autoCreateTbl @@ -4046,9 +4054,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (dataSource && dataSource->type == cJSON_String && dataSource->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, - dataSource->valuestring, TSDB_DB_NAME_LEN); + dataSource->valuestring, + min(SMALL_BUFF_LEN, strlen(dataSource->valuestring) + 1)); } else if (!dataSource) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN); + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", + min(SMALL_BUFF_LEN, strlen("rand") + 1)); } else { errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__); @@ -4129,9 +4139,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, - sampleFormat->valuestring, TSDB_DB_NAME_LEN); + sampleFormat->valuestring, + min(SMALL_BUFF_LEN, + strlen(sampleFormat->valuestring) + 1)); } else if (!sampleFormat) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN); + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", + SMALL_BUFF_LEN); } else { printf("ERROR: failed to read json, sample_format not found\n"); goto PARSE_OVER; @@ -4141,9 +4154,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, - sampleFile->valuestring, MAX_FILE_NAME_LEN); + sampleFile->valuestring, + min(MAX_FILE_NAME_LEN, + strlen(sampleFile->valuestring) + 1)); } else if (!sampleFile) { - memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); + memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, + MAX_FILE_NAME_LEN); } else { printf("ERROR: failed to read json, sample_file not found\n"); goto PARSE_OVER; @@ -4383,10 +4399,14 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode"); - if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) { - tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE); + if (queryMode + && queryMode->type == cJSON_String + && queryMode->valuestring != NULL) { + tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, + min(SMALL_BUFF_LEN, strlen(queryMode->valuestring) + 1)); } else if (!queryMode) { - tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); + tstrncpy(g_queryInfo.queryMode, "taosc", + min(SMALL_BUFF_LEN, strlen("taosc") + 1)); } else { printf("ERROR: failed to read json, query_mode not found\n"); goto PARSE_OVER; @@ -6549,9 +6569,10 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { } // num_of_DPT if ((g_args.verbose_print) && - (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) && - (0 == strncasecmp( - superTblInfo->dataSource, "sample", strlen("sample")))) { + (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) + && (0 == strncasecmp( + superTblInfo->dataSource, + "sample", strlen("sample")))) { verbosePrint("%s() LN%d samplePos=%"PRId64"\n", __func__, __LINE__, pThreadInfo->samplePos); } @@ -7389,7 +7410,7 @@ static void *specifiedTableQuery(void *sarg) { static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) { char sourceString[32] = "xxxx"; - char subTblName[MAX_TB_NAME_SIZE*3]; + char subTblName[TSDB_TABLE_NAME_LEN]; sprintf(subTblName, "%s.%s", g_queryInfo.dbName, g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); @@ -7548,8 +7569,8 @@ static int queryTestProcess() { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); + char sqlStr[TSDB_DB_NAME_LEN + 5]; + sprintf(sqlStr, "USE %s", g_queryInfo.dbName); if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(taos); free(infos); @@ -7738,8 +7759,8 @@ static void *superSubscribe(void *sarg) { } } - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); + char sqlStr[TSDB_DB_NAME_LEN + 5]; + sprintf(sqlStr, "USE %s", g_queryInfo.dbName); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); errorPrint( "use database %s failed!\n\n", @@ -7880,8 +7901,8 @@ static void *specifiedSubscribe(void *sarg) { } } - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); + char sqlStr[TSDB_DB_NAME_LEN + 5]; + sprintf(sqlStr, "USE %s", g_queryInfo.dbName); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); return NULL; @@ -8176,7 +8197,7 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN); g_Dbs.db[0].dbCfg.replica = g_args.replica; - tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8); + tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", SMALL_BUFF_LEN); tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); @@ -8209,8 +8230,8 @@ static void setParaFromArg() { g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, - g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); - tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + g_args.tb_prefix, TBNAME_PREFIX_LEN); + tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", SMALL_BUFF_LEN); if (g_args.iface == INTERFACE_BUT) { g_Dbs.db[0].superTbls[0].iface = TAOSC_IFACE; @@ -8231,7 +8252,7 @@ static void setParaFromArg() { } tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - data_type[i], strlen(data_type[i]) + 1); + data_type[i], min(DATATYPE_BUFF_LEN, strlen(data_type[i]) + 1)); g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].columnCount++; } @@ -8242,18 +8263,18 @@ static void setParaFromArg() { for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - "INT", strlen("INT") + 1); + "INT", min(DATATYPE_BUFF_LEN, strlen("INT") + 1)); g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; g_Dbs.db[0].superTbls[0].columnCount++; } } tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, - "INT", strlen("INT") + 1); + "INT", min(DATATYPE_BUFF_LEN, strlen("INT") + 1)); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, - "BINARY", strlen("BINARY") + 1); + "BINARY", min(DATATYPE_BUFF_LEN, strlen("BINARY") + 1)); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { @@ -8389,7 +8410,7 @@ static void queryResult() { pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; tstrncpy(pThreadInfo->tb_prefix, - g_Dbs.db[0].superTbls[0].childTblPrefix, TSDB_TABLE_NAME_LEN - 20); + g_Dbs.db[0].superTbls[0].childTblPrefix, TBNAME_PREFIX_LEN); } else { pThreadInfo->ntables = g_args.num_of_tables; pThreadInfo->end_table_to = g_args.num_of_tables -1; From 755f666e2a36052250cf544fcb6eccce92b51034 Mon Sep 17 00:00:00 2001 From: wpan Date: Fri, 30 Jul 2021 14:30:32 +0800 Subject: [PATCH 3/5] fix bug --- src/query/src/qExecutor.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9d516e976f..6b32508f07 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2848,9 +2848,17 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQuery); - needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowCellInfoOffset); + if (pQuery->interpQuery) { + needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset); + } else { + if (setWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } } else if (pQuery->stableQuery && (!isTsCompQuery(pQuery))) { // stable aggregate, not interval aggregate or normal column aggregate doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->rowCellInfoOffset, pTableScanInfo->numOfOutput, From 310ef268291d0567f2a3de33a953a5b695c67aeb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 1 Aug 2021 10:33:28 +0800 Subject: [PATCH 4/5] [TD-5625]: taosdemo don't use printf for data generation. (#7094) * [TD-5625]: taosdemo don't use printf for data generation. * re cherry-pick --- src/kit/taosdemo/taosdemo.c | 397 ++++++++++++++++++++++++++---------- 1 file changed, 284 insertions(+), 113 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index a7a4567b97..0a97402bdb 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -82,9 +82,18 @@ extern char configDir[]; #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ -#define MAX_PREPARED_RAND 1000000 #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. +#define MAX_PREPARED_RAND 1000000 +#define INT_BUFF_LEN 11 +#define BIGINT_BUFF_LEN 21 +#define SMALLINT_BUFF_LEN 6 +#define TINYINT_BUFF_LEN 4 +#define BOOL_BUFF_LEN 6 +#define FLOAT_BUFF_LEN 22 +#define DOUBLE_BUFF_LEN 42 +#define TIMESTAMP_BUFF_LEN 21 + #define MAX_SAMPLES_ONCE_FROM_FILE 10000 #define MAX_NUM_COLUMNS (TSDB_MAX_COLUMNS - 1) // exclude first column timestamp @@ -561,11 +570,23 @@ static void init_rand_data(); /* ************ Global variables ************ */ -int32_t randint[MAX_PREPARED_RAND]; -int64_t randbigint[MAX_PREPARED_RAND]; -float randfloat[MAX_PREPARED_RAND]; -double randdouble[MAX_PREPARED_RAND]; -char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", +int32_t g_randint[MAX_PREPARED_RAND]; +int64_t g_randbigint[MAX_PREPARED_RAND]; +float g_randfloat[MAX_PREPARED_RAND]; +double g_randdouble[MAX_PREPARED_RAND]; + +char *g_randbool_buff = NULL; +char *g_randint_buff = NULL; +char *g_rand_voltage_buff = NULL; +char *g_randbigint_buff = NULL; +char *g_randsmallint_buff = NULL; +char *g_randtinyint_buff = NULL; +char *g_randfloat_buff = NULL; +char *g_rand_current_buff = NULL; +char *g_rand_phase_buff = NULL; +char *g_randdouble_buff = NULL; + +char *g_aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; #define DEFAULT_DATATYPE_NUM 3 @@ -1307,67 +1328,144 @@ static void selectAndGetResult( } } +static char *rand_bool_str(){ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randbool_buff + (cursor * BOOL_BUFF_LEN); +} + static int32_t rand_bool(){ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 2; + return g_randint[cursor] % 2; } -static int32_t rand_tinyint(){ +static char *rand_tinyint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randtinyint_buff + (cursor * TINYINT_BUFF_LEN); +} + +static int32_t rand_tinyint() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 128; + return g_randint[cursor] % 128; } -static int32_t rand_smallint(){ +static char *rand_smallint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randsmallint_buff + (cursor * SMALLINT_BUFF_LEN); +} + +static int32_t rand_smallint() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 32767; + return g_randint[cursor] % 32767; } -static int32_t rand_int(){ +static char *rand_int_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randint_buff + (cursor * INT_BUFF_LEN); +} + +static int32_t rand_int() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor]; + return g_randint[cursor]; } -static int64_t rand_bigint(){ +static char *rand_bigint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randbigint_buff + (cursor * BIGINT_BUFF_LEN); +} + +static int64_t rand_bigint() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randbigint[cursor]; + return g_randbigint[cursor]; } -static float rand_float(){ +static char *rand_float_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randfloat_buff + (cursor * FLOAT_BUFF_LEN); +} + +static float rand_float() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randfloat[cursor]; + return g_randfloat[cursor]; } -static float demo_current_float(){ +static char *demo_current_float_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_rand_current_buff + (cursor * FLOAT_BUFF_LEN); +} + +static float UNUSED_FUNC demo_current_float() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return (float)(9.8 + 0.04 * (randint[cursor] % 10) + randfloat[cursor]/1000000000); + return (float)(9.8 + 0.04 * (g_randint[cursor] % 10) + g_randfloat[cursor]/1000000000); } -static int32_t demo_voltage_int(){ +static char *demo_voltage_int_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_rand_voltage_buff + (cursor * INT_BUFF_LEN); +} + +static int32_t UNUSED_FUNC demo_voltage_int() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return 215 + randint[cursor] % 10; + return 215 + g_randint[cursor] % 10; } -static float demo_phase_float(){ +static char *demo_phase_float_str() { + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_rand_phase_buff + (cursor * FLOAT_BUFF_LEN); +} + +static float UNUSED_FUNC demo_phase_float(){ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return (float)((115 + randint[cursor] % 10 + randfloat[cursor]/1000000000)/360); + return (float)((115 + g_randint[cursor] % 10 + g_randfloat[cursor]/1000000000)/360); } #if 0 @@ -1402,19 +1500,76 @@ static void rand_string(char *str, int size) { } } -static double rand_double() { +static char *rand_double_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randdouble_buff + (cursor * DOUBLE_BUFF_LEN); +} + +static double rand_double() +{ static int cursor; cursor++; cursor = cursor % MAX_PREPARED_RAND; - return randdouble[cursor]; + return g_randdouble[cursor]; } static void init_rand_data() { + + g_randint_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randint_buff); + g_rand_voltage_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_rand_voltage_buff); + g_randbigint_buff = calloc(1, BIGINT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randbigint_buff); + g_randsmallint_buff = calloc(1, SMALLINT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randsmallint_buff); + g_randtinyint_buff = calloc(1, TINYINT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randtinyint_buff); + g_randbool_buff = calloc(1, BOOL_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randbool_buff); + g_randfloat_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randfloat_buff); + g_rand_current_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_rand_current_buff); + g_rand_phase_buff = calloc(1, FLOAT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_rand_phase_buff); + g_randdouble_buff = calloc(1, DOUBLE_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randdouble_buff); + for (int i = 0; i < MAX_PREPARED_RAND; i++){ - randint[i] = (int)(taosRandom() % 65535); - randbigint[i] = (int64_t)(taosRandom() % 2147483648); - randfloat[i] = (float)(taosRandom() / 1000.0); - randdouble[i] = (double)(taosRandom() / 1000000.0); + g_randint[i] = (int)(taosRandom() % 65535); + sprintf(g_randint_buff + i * INT_BUFF_LEN, "%d", + g_randint[i]); + sprintf(g_rand_voltage_buff + i * INT_BUFF_LEN, "%d", + 215 + g_randint[i] % 10); + + sprintf(g_randbool_buff + i * BOOL_BUFF_LEN, "%s", + ((g_randint[i] % 2) & 1)?"true":"false"); + sprintf(g_randsmallint_buff + i * SMALLINT_BUFF_LEN, "%d", + g_randint[i] % 32767); + sprintf(g_randtinyint_buff + i * TINYINT_BUFF_LEN, "%d", + g_randint[i] % 128); + + g_randbigint[i] = (int64_t)(taosRandom() % 2147483648); + sprintf(g_randbigint_buff + i * BIGINT_BUFF_LEN, "%"PRId64"", + g_randbigint[i]); + + g_randfloat[i] = (float)(taosRandom() / 1000.0); + sprintf(g_randfloat_buff + i * FLOAT_BUFF_LEN, "%f", + g_randfloat[i]); + sprintf(g_rand_current_buff + i * FLOAT_BUFF_LEN, "%f", + (float)(9.8 + 0.04 * (g_randint[i] % 10) + + g_randfloat[i]/1000000000)); + sprintf(g_rand_phase_buff + i * FLOAT_BUFF_LEN, "%f", + (float)((115 + g_randint[i] % 10 + + g_randfloat[i]/1000000000)/360)); + + g_randdouble[i] = (double)(taosRandom() / 1000000.0); + sprintf(g_randdouble_buff + i * DOUBLE_BUFF_LEN, "%f", + g_randdouble[i]); } } @@ -2496,21 +2651,21 @@ static int calcRowLen(SSuperTable* superTbls) { } else if (strcasecmp(dataType, "NCHAR") == 0) { lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { - lenOfOneRow += 11; + lenOfOneRow += INT_BUFF_LEN; } else if (strcasecmp(dataType, "BIGINT") == 0) { - lenOfOneRow += 21; + lenOfOneRow += BIGINT_BUFF_LEN; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - lenOfOneRow += 6; + lenOfOneRow += SMALLINT_BUFF_LEN; } else if (strcasecmp(dataType, "TINYINT") == 0) { - lenOfOneRow += 4; + lenOfOneRow += TINYINT_BUFF_LEN; } else if (strcasecmp(dataType, "BOOL") == 0) { - lenOfOneRow += 6; + lenOfOneRow += BOOL_BUFF_LEN; } else if (strcasecmp(dataType, "FLOAT") == 0) { - lenOfOneRow += 22; + lenOfOneRow += FLOAT_BUFF_LEN; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - lenOfOneRow += 42; + lenOfOneRow += DOUBLE_BUFF_LEN; } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { - lenOfOneRow += 21; + lenOfOneRow += TIMESTAMP_BUFF_LEN; } else { printf("get error data type : %s\n", dataType); exit(-1); @@ -2529,19 +2684,19 @@ static int calcRowLen(SSuperTable* superTbls) { } else if (strcasecmp(dataType, "NCHAR") == 0) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + INT_BUFF_LEN; } else if (strcasecmp(dataType, "BIGINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + BIGINT_BUFF_LEN; } else if (strcasecmp(dataType, "SMALLINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + SMALLINT_BUFF_LEN; } else if (strcasecmp(dataType, "TINYINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + TINYINT_BUFF_LEN; } else if (strcasecmp(dataType, "BOOL") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + BOOL_BUFF_LEN; } else if (strcasecmp(dataType, "FLOAT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + FLOAT_BUFF_LEN; } else if (strcasecmp(dataType, "DOUBLE") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + DOUBLE_BUFF_LEN; } else { printf("get error tag type : %s\n", dataType); exit(-1); @@ -2770,21 +2925,21 @@ static int createSuperTable( } else { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "INT"); } - lenOfOneRow += 11; + lenOfOneRow += INT_BUFF_LEN; } else if (strcasecmp(dataType, "BIGINT") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "BIGINT"); - lenOfOneRow += 21; + lenOfOneRow += BIGINT_BUFF_LEN; } else if (strcasecmp(dataType, "SMALLINT") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "SMALLINT"); - lenOfOneRow += 6; + lenOfOneRow += SMALLINT_BUFF_LEN; } else if (strcasecmp(dataType, "TINYINT") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "TINYINT"); - lenOfOneRow += 4; + lenOfOneRow += TINYINT_BUFF_LEN; } else if (strcasecmp(dataType, "BOOL") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "BOOL"); - lenOfOneRow += 6; + lenOfOneRow += BOOL_BUFF_LEN; } else if (strcasecmp(dataType, "FLOAT") == 0) { if (g_args.demo_mode) { if (colIndex == 0) { @@ -2796,15 +2951,15 @@ static int createSuperTable( len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "FLOAT"); } - lenOfOneRow += 22; + lenOfOneRow += FLOAT_BUFF_LEN; } else if (strcasecmp(dataType, "DOUBLE") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "DOUBLE"); - lenOfOneRow += 42; + lenOfOneRow += DOUBLE_BUFF_LEN; } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", colIndex, "TIMESTAMP"); - lenOfOneRow += 21; + lenOfOneRow += TIMESTAMP_BUFF_LEN; } else { taos_close(taos); errorPrint("%s() LN%d, config error data type : %s\n", @@ -2867,31 +3022,31 @@ static int createSuperTable( len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "INT"); } - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + INT_BUFF_LEN; } else if (strcasecmp(dataType, "BIGINT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "BIGINT"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + BIGINT_BUFF_LEN; } else if (strcasecmp(dataType, "SMALLINT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "SMALLINT"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + SMALLINT_BUFF_LEN; } else if (strcasecmp(dataType, "TINYINT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "TINYINT"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + TINYINT_BUFF_LEN; } else if (strcasecmp(dataType, "BOOL") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "BOOL"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + BOOL_BUFF_LEN; } else if (strcasecmp(dataType, "FLOAT") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "FLOAT"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + FLOAT_BUFF_LEN; } else if (strcasecmp(dataType, "DOUBLE") == 0) { len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "DOUBLE"); - lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42; + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + DOUBLE_BUFF_LEN; } else { taos_close(taos); errorPrint("%s() LN%d, config error tag type : %s\n", @@ -4894,6 +5049,17 @@ static void postFreeResource() { } } } + + tmfree(g_randbool_buff); + tmfree(g_randint_buff); + tmfree(g_rand_voltage_buff); + tmfree(g_randbigint_buff); + tmfree(g_randsmallint_buff); + tmfree(g_randtinyint_buff); + tmfree(g_randfloat_buff); + tmfree(g_rand_current_buff); + tmfree(g_rand_phase_buff); + tmfree(g_randdouble_buff); } static int getRowDataFromSample( @@ -4956,56 +5122,61 @@ static int64_t generateStbRowData( rand_string(buf, stbInfo->columns[i].dataLen); dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf); tmfree(buf); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "INT", strlen("INT"))) { - if ((g_args.demo_mode) && (i == 1)) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", demo_voltage_int()); - } else { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", rand_int()); - } - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "BIGINT", strlen("BIGINT"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%"PRId64",", rand_bigint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "FLOAT", strlen("FLOAT"))) { - if (g_args.demo_mode) { - if (i == 0) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%f,", demo_current_float()); + } else { + char *tmp; + + if (0 == strncasecmp(stbInfo->columns[i].dataType, + "INT", strlen("INT"))) { + if ((g_args.demo_mode) && (i == 1)) { + tmp = demo_voltage_int_str(); } else { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%f,", demo_phase_float()); + tmp = rand_int_str(); } - } else { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%f,", rand_float()); + tstrncpy(pstr + dataLen, tmp, INT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "BIGINT", strlen("BIGINT"))) { + tmp = rand_bigint_str(); + tstrncpy(pstr + dataLen, tmp, BIGINT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "FLOAT", strlen("FLOAT"))) { + if (g_args.demo_mode) { + if (i == 0) { + tmp = demo_current_float_str(); + } else { + tmp = demo_phase_float_str(); + } + } else { + tmp = rand_float_str(); + } + tstrncpy(pstr + dataLen, tmp, FLOAT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "DOUBLE", strlen("DOUBLE"))) { + tmp = rand_double_str(); + tstrncpy(pstr + dataLen, tmp, DOUBLE_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "SMALLINT", strlen("SMALLINT"))) { + tmp = rand_smallint_str(); + tstrncpy(pstr + dataLen, tmp, SMALLINT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "TINYINT", strlen("TINYINT"))) { + tmp = rand_tinyint_str(); + tstrncpy(pstr + dataLen, tmp, TINYINT_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "BOOL", strlen("BOOL"))) { + tmp = rand_bool_str(); + tstrncpy(pstr + dataLen, tmp, BOOL_BUFF_LEN); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "TIMESTAMP", strlen("TIMESTAMP"))) { + tmp = rand_int_str(); + tstrncpy(pstr + dataLen, tmp, INT_BUFF_LEN); + } else { + errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType); + return -1; } - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "DOUBLE", strlen("DOUBLE"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%f,", rand_double()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "SMALLINT", strlen("SMALLINT"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", rand_smallint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "TINYINT", strlen("TINYINT"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", rand_tinyint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "BOOL", strlen("BOOL"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%d,", rand_bool()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, - "TIMESTAMP", strlen("TIMESTAMP"))) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, - "%"PRId64",", rand_bigint()); - } else { - errorPrint( "Not support data type: %s\n", stbInfo->columns[i].dataType); - return -1; + + dataLen += strlen(tmp); + tstrncpy(pstr + dataLen, ",", 2); + dataLen += 1; } } @@ -7105,7 +7276,7 @@ static void *readTable(void *sarg) { int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; - int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + int n = do_aggreFunc ? (sizeof(g_aggreFunc) / sizeof(g_aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } @@ -7117,7 +7288,7 @@ static void *readTable(void *sarg) { uint64_t count = 0; for (int64_t i = 0; i < num_of_tables; i++) { sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64, - aggreFunc[j], tb_prefix, i, sTime); + g_aggreFunc[j], tb_prefix, i, sTime); double t = taosGetTimestampMs(); TAOS_RES *pSql = taos_query(taos, command); @@ -7142,9 +7313,9 @@ static void *readTable(void *sarg) { } fprintf(fp, "|%10s | %"PRId64" | %12.2f | %10.2f |\n", - aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, + g_aggreFunc[j][0] == '*' ? " * " : g_aggreFunc[j], totalData, (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); - printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000); + printf("select %10s took %.6f second(s)\n", g_aggreFunc[j], totalT * 1000); } fprintf(fp, "\n"); fclose(fp); @@ -7169,7 +7340,7 @@ static void *readMetric(void *sarg) { int64_t totalData = num_of_DPT * num_of_tables; bool do_aggreFunc = g_Dbs.do_aggreFunc; - int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + int n = do_aggreFunc ? (sizeof(g_aggreFunc) / sizeof(g_aggreFunc[0])) : 2; if (!do_aggreFunc) { printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); } @@ -7190,7 +7361,7 @@ static void *readMetric(void *sarg) { } strncat(condition, tempS, COND_BUF_LEN - 1); - sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); + sprintf(command, "select %s from meters where %s", g_aggreFunc[j], condition); printf("Where condition: %s\n", condition); fprintf(fp, "%s\n", command); @@ -7215,7 +7386,7 @@ static void *readMetric(void *sarg) { fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", num_of_tables * num_of_DPT / (t * 1000.0), t); - printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0); + printf("select %10s took %.6f second(s)\n\n", g_aggreFunc[j], t * 1000.0); taos_free_result(pSql); } From ede06b75aa874ecb18d84e0ad51a760f9051f06c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 1 Aug 2021 13:08:14 +0800 Subject: [PATCH 5/5] Hotfix/sangshuduo/td 3197 taosdemo coverity scan for master (#7099) * [TD-3197]: taosdemo and taosdump coverity scan issues. * exit if read sample file failed. * fix converity scan issue. * fix coverity scan issue. * fix coverity scan memory leak. * fix resource leak reported by coverity scan. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0a97402bdb..773506c5ae 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -6163,6 +6163,7 @@ static int32_t prepareStbStmt( if (0 != ret) { errorPrint("%s() LN%d, stmt_bind_param() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); + free(bindArray); return -1; } // if msg > 3MB, break @@ -6170,6 +6171,7 @@ static int32_t prepareStbStmt( if (0 != ret) { errorPrint("%s() LN%d, stmt_add_batch() failed! reason: %s\n", __func__, __LINE__, taos_stmt_errstr(stmt)); + free(bindArray); return -1; }