xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #1266 from taosdata/feature/liaohj 

Feature/liaohj
This commit is contained in:
slguan 2020-02-27 11:02:32 +08:00 committed by GitHub
parent e42d6fa100 3932d0accd
commit 1a8025666f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 3034 additions and 3083 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/client/inc/tscSQLParser.h
View File

@ -86,15 +86,16 @@ enum _sql_cmd {
TSDB_SQL_MAX //48 TSDB_SQL_MAX //48
}; };
#define MAX_TOKEN_LEN 30
// token type
enum { enum {
TSQL_NODE_TYPE_EXPR = 0x1, TSQL_NODE_TYPE_EXPR = 0x1,
TSQL_NODE_TYPE_ID = 0x2, TSQL_NODE_TYPE_ID = 0x2,
TSQL_NODE_TYPE_VALUE = 0x4, TSQL_NODE_TYPE_VALUE = 0x4,
}; };
#define NON_ARITHMEIC_EXPR 0
#define NORMAL_ARITHMETIC 1
#define AGG_ARIGHTMEIC 2
extern char tTokenTypeSwitcher[13]; extern char tTokenTypeSwitcher[13];
#define toTSDBType(x) \ #define toTSDBType(x) \
@ -112,7 +113,7 @@ typedef struct SLimitVal {
} SLimitVal; } SLimitVal;
typedef struct SOrderVal { typedef struct SOrderVal {
int32_t order; uint32_t order;
int32_t orderColId; int32_t orderColId;
} SOrderVal; } SOrderVal;

5
src/client/inc/tscUtil.h
View File

@ -128,6 +128,8 @@ void tscFieldInfoSetValFromSchema(SFieldInfo* pFieldInfo, int32_t index, SSchema
void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* pField); void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* pField);
void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, const char* name, int16_t bytes); void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, const char* name, int16_t bytes);
void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible); void tscFieldInfoUpdateVisible(SFieldInfo* pFieldInfo, int32_t index, bool visible);
void tscFieldInfoSetExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlExpr* pExpr);
void tscFieldInfoSetBinExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlFunctionExpr* pExpr);
void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo); void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo);
void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo); void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo);
@ -149,9 +151,10 @@ SSqlExpr* tscSqlExprInsertEmpty(SQueryInfo* pQueryInfo, int32_t index, int16_t f
SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type,
int16_t size); int16_t size);
int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo);
SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index); SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index);
void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t uid); void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t uid, bool deepcopy);
void* tscSqlExprDestroy(SSqlExpr* pExpr); void* tscSqlExprDestroy(SSqlExpr* pExpr);
void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo); void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo);

89
src/client/inc/tsclient.h
View File

@ -31,9 +31,8 @@ extern "C" {
#include "tsqlfunction.h" #include "tsqlfunction.h"
#include "tutil.h" #include "tutil.h"
#define TSC_GET_RESPTR_BASE(res, _queryinfo, col, ord) \ #define TSC_GET_RESPTR_BASE(res, _queryinfo, col) (res->data + ((_queryinfo)->fieldsInfo.pSqlExpr[col]->offset) * res->numOfRows)
(res->data + tscFieldInfoGetOffset(_queryinfo, col) * res->numOfRows)
// forward declaration // forward declaration
struct SSqlInfo; struct SSqlInfo;
@ -70,13 +69,19 @@ typedef struct SSqlExpr {
int16_t interResBytes; // inter result buffer size int16_t interResBytes; // inter result buffer size
int16_t numOfParams; // argument value of each function int16_t numOfParams; // argument value of each function
tVariant param[3]; // parameters are not more than 3 tVariant param[3]; // parameters are not more than 3
int32_t offset; // sub result column value of arithmetic expression.
} SSqlExpr; } SSqlExpr;
typedef struct SColumnIndex {
int16_t tableIndex;
int16_t columnIndex;
} SColumnIndex;
typedef struct SFieldInfo { typedef struct SFieldInfo {
int16_t numOfOutputCols; // number of column in result int16_t numOfOutputCols; // number of column in result
int16_t numOfAlloc; // allocated size int16_t numOfAlloc; // allocated size
TAOS_FIELD *pFields; TAOS_FIELD *pFields;
short * pOffset; // short * pOffset;
/* /*
* define if this column is belong to the queried result, it may be add by parser to faciliate * define if this column is belong to the queried result, it may be add by parser to faciliate
@ -85,20 +90,17 @@ typedef struct SFieldInfo {
* NOTE: these hidden columns always locate at the end of the output columns * NOTE: these hidden columns always locate at the end of the output columns
*/ */
bool * pVisibleCols; bool * pVisibleCols;
int32_t numOfHiddenCols; // the number of column not belongs to the queried result columns int32_t numOfHiddenCols; // the number of column not belongs to the queried result columns
SSqlFunctionExpr** pExpr; // used for aggregation arithmetic express,such as count(*)+count(*)
SSqlExpr** pSqlExpr;
} SFieldInfo; } SFieldInfo;
typedef struct SSqlExprInfo { typedef struct SSqlExprInfo {
int16_t numOfAlloc; int16_t numOfAlloc;
int16_t numOfExprs; int16_t numOfExprs;
SSqlExpr *pExprs; SSqlExpr** pExprs;
} SSqlExprInfo; } SSqlExprInfo;
typedef struct SColumnIndex {
int16_t tableIndex;
int16_t columnIndex;
} SColumnIndex;
typedef struct SColumnBase { typedef struct SColumnBase {
SColumnIndex colIndex; SColumnIndex colIndex;
int32_t numOfFilters; int32_t numOfFilters;
@ -163,7 +165,7 @@ typedef struct STableDataBlocks {
int32_t rowSize; // row size for current table int32_t rowSize; // row size for current table
uint32_t nAllocSize; uint32_t nAllocSize;
uint32_t headerSize; // header for metadata (submit metadata) uint32_t headerSize; // header for metadata (submit metadata)
uint32_t size; uint32_t size;
/* /*
@ -198,9 +200,9 @@ typedef struct SQueryInfo {
char intervalTimeUnit; char intervalTimeUnit;
int64_t etime, stime; int64_t etime, stime;
int64_t nAggTimeInterval; // aggregation time interval int64_t intervalTime; // aggregation time interval
int64_t nSlidingTime; // sliding window in mseconds int64_t slidingTime; // sliding window in mseconds
SSqlGroupbyExpr groupbyExpr; // group by tags info SSqlGroupbyExpr groupbyExpr; // group by tags info
SColumnBaseInfo colList; SColumnBaseInfo colList;
SFieldInfo fieldsInfo; SFieldInfo fieldsInfo;
@ -216,9 +218,9 @@ typedef struct SQueryInfo {
int64_t * defaultVal; // default value for interpolation int64_t * defaultVal; // default value for interpolation
char * msg; // pointer to the pCmd->payload to keep error message temporarily char * msg; // pointer to the pCmd->payload to keep error message temporarily
int64_t clauseLimit; // limit for current sub clause int64_t clauseLimit; // limit for current sub clause
// offset value in the original sql expression, NOT sent to virtual node, only applied at client side // offset value in the original sql expression, NOT sent to virtual node, only applied at client side
int64_t prjOffset; int64_t prjOffset;
} SQueryInfo; } SQueryInfo;
// data source from sql string or from file // data source from sql string or from file
@ -269,29 +271,28 @@ typedef struct SResRec {
struct STSBuf; struct STSBuf;
typedef struct { typedef struct {
uint8_t code; uint8_t code;
int64_t numOfRows; // num of results in current retrieved int64_t numOfRows; // num of results in current retrieved
int64_t numOfTotal; // num of total results int64_t numOfTotal; // num of total results
int64_t numOfTotalInCurrentClause; // num of total result in current subclause int64_t numOfTotalInCurrentClause; // num of total result in current subclause
char * pRsp;
char * pRsp; int rspType;
int rspType; int rspLen;
int rspLen; uint64_t qhandle;
uint64_t qhandle; int64_t uid;
int64_t uid; int64_t useconds;
int64_t useconds; int64_t offset; // offset value from vnode during projection query of stable
int64_t offset; // offset value from vnode during projection query of stable int row;
int row; int16_t numOfCols;
int16_t numOfnchar; int16_t precision;
int16_t precision; int32_t numOfGroups;
int32_t numOfGroups; SResRec * pGroupRec;
SResRec * pGroupRec; char * data;
char * data; void ** tsrow;
short * bytes; char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t)
void ** tsrow; SColumnIndex *pColumnIndex;
char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t)
struct SLocalReducer *pLocalReducer; struct SLocalReducer *pLocalReducer;
SColumnIndex * pColumnIndex;
} SSqlRes; } SSqlRes;
typedef struct _tsc_obj { typedef struct _tsc_obj {
@ -404,19 +405,17 @@ int taos_retrieve(TAOS_RES *res);
int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo *pQueryInfo);
void tscRestoreSQLFunctionForMetricQuery(SQueryInfo *pQueryInfo); void tscRestoreSQLFunctionForMetricQuery(SQueryInfo *pQueryInfo);
void tscClearSqlMetaInfoForce(SSqlCmd *pCmd);
int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo);
void tscDestroyResPointerInfo(SSqlRes *pRes); void tscDestroyResPointerInfo(SSqlRes *pRes);
void tscFreeSqlCmdData(SSqlCmd *pCmd); void tscFreeSqlCmdData(SSqlCmd *pCmd);
void tscFreeResData(SSqlObj* pSql); void tscFreeResData(SSqlObj *pSql);
/** /**
* free query result of the sql object * free query result of the sql object
* @param pObj * @param pObj
*/ */
void tscFreeSqlResult(SSqlObj* pSql); void tscFreeSqlResult(SSqlObj *pSql);
/** /**
* only free part of resources allocated during query. * only free part of resources allocated during query.

3
src/client/src/tscAst.c
View File

@ -158,7 +158,7 @@ static tSQLSyntaxNode *tSQLSyntaxNodeCreate(SSchema *pSchema, int32_t numOfCols,
return pNode; return pNode;
} }
static uint8_t getBinaryExprOptr(SSQLToken *pToken) { uint8_t getBinaryExprOptr(SSQLToken *pToken) {
switch (pToken->type) { switch (pToken->type) {
case TK_LT: case TK_LT:
return TSDB_RELATION_LESS; return TSDB_RELATION_LESS;
@ -183,6 +183,7 @@ static uint8_t getBinaryExprOptr(SSQLToken *pToken) {
case TK_STAR: case TK_STAR:
return TSDB_BINARY_OP_MULTIPLY; return TSDB_BINARY_OP_MULTIPLY;
case TK_SLASH: case TK_SLASH:
case TK_DIVIDE:
return TSDB_BINARY_OP_DIVIDE; return TSDB_BINARY_OP_DIVIDE;
case TK_REM: case TK_REM:
return TSDB_BINARY_OP_REMAINDER; return TSDB_BINARY_OP_REMAINDER;

18
src/client/src/tscAsync.c
View File

@ -283,8 +283,15 @@ void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows) {
return; return;
} }
for (int i = 0; i < pCmd->numOfCols; ++i) for (int i = 0; i < pCmd->numOfCols; ++i){
pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i, pQueryInfo->order) + pRes->bytes[i] * pRes->row; SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i];
if (pExpr != NULL) {
pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row;
} else {
//todo add
}
}
pRes->row++; pRes->row++;
(*pSql->fetchFp)(pSql->param, pSql, pSql->res.tsrow); (*pSql->fetchFp)(pSql->param, pSql, pSql->res.tsrow);
@ -298,7 +305,12 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
for (int i = 0; i < pCmd->numOfCols; ++i) { for (int i = 0; i < pCmd->numOfCols; ++i) {
pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i, pQueryInfo->order) + pRes->bytes[i] * pRes->row; SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i];
if (pExpr != NULL) {
pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row;
} else {
//todo add
}
} }
pRes->row++; pRes->row++;

41
src/client/src/tscFunctionImpl.c
View File

@ -322,6 +322,10 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool stableQueryFunctChanged(int32_t funcId) {
return (aAggs[funcId].stableFuncId != funcId);
}
/** /**
* the numOfRes should be kept, since it may be used later * the numOfRes should be kept, since it may be used later
* and allow the ResultInfo to be re initialized * and allow the ResultInfo to be re initialized
@ -719,12 +723,15 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY
return BLK_DATA_NO_NEEDED; return BLK_DATA_NO_NEEDED;
} }
SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); // result buffer has not been set yet.
if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED;
return BLK_DATA_ALL_NEEDED; //todo optimize the filter info
} else { // data in current block is not earlier than current result // SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes);
return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; // if (pInfo->hasResult != DATA_SET_FLAG) {
} // return BLK_DATA_ALL_NEEDED;
// } else { // data in current block is not earlier than current result
// return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
// }
} }
static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId, static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId,
@ -733,12 +740,13 @@ static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY
return BLK_DATA_NO_NEEDED; return BLK_DATA_NO_NEEDED;
} }
SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); return BLK_DATA_ALL_NEEDED;
if (pInfo->hasResult != DATA_SET_FLAG) { // SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes);
return BLK_DATA_ALL_NEEDED; // if (pInfo->hasResult != DATA_SET_FLAG) {
} else { // return BLK_DATA_ALL_NEEDED;
return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; // } else {
} // return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
// }
} }
////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////
@ -1437,7 +1445,9 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) {
*/ */
pStd->stage++; pStd->stage++;
avg_finalizer(pCtx); avg_finalizer(pCtx);
pResInfo->initialized = true; // set it initialized to avoid re-initialization
// save average value into tmpBuf, for second stage scan // save average value into tmpBuf, for second stage scan
SAvgInfo *pAvg = pResInfo->interResultBuf; SAvgInfo *pAvg = pResInfo->interResultBuf;
@ -2184,7 +2194,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
// only the first_stage_merge is directly written data into final output buffer // only the first_stage_merge is directly written data into final output buffer
if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
return (STopBotInfo*) pCtx->aOutputBuf; return (STopBotInfo*) pCtx->aOutputBuf;
} else { // for normal table query and super table at the secondary_stage, result is written to intermediate buffer } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer
return pResInfo->interResultBuf; return pResInfo->interResultBuf;
} }
} }
@ -3312,7 +3322,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) {
tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order,
arithmetic_callback_function); arithmetic_callback_function);
pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size/* * GET_FORWARD_DIRECTION_FACTOR(pCtx->order)*/; pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size;
pCtx->param[1].pz = NULL; pCtx->param[1].pz = NULL;
} }
@ -3573,6 +3583,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
} }
GET_RES_INFO(pCtx)->numOfRes = 1; // todo add test case GET_RES_INFO(pCtx)->numOfRes = 1; // todo add test case
doFinalizer(pCtx);
} }
/* /*

17
src/client/src/tscJoinProcess.c
View File

@ -100,7 +100,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSubquerySupporter* pSuppor
* in case of stable query, limit/offset is not applied here. the limit/offset is applied to the * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
* final results which is acquired after the secondry merge of in the client. * final results which is acquired after the secondry merge of in the client.
*/ */
if (pLimit->offset == 0 || pQueryInfo->nAggTimeInterval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { if (pLimit->offset == 0 || pQueryInfo->intervalTime > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
if (*st > elem1.ts) { if (*st > elem1.ts) {
*st = elem1.ts; *st = elem1.ts;
} }
@ -165,7 +165,7 @@ SJoinSubquerySupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pS
pSupporter->subqueryIndex = index; pSupporter->subqueryIndex = index;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
pSupporter->interval = pQueryInfo->nAggTimeInterval; pSupporter->interval = pQueryInfo->intervalTime;
pSupporter->limit = pQueryInfo->limit; pSupporter->limit = pQueryInfo->limit;
SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, pSql->cmd.clauseIndex, index); SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pSql->cmd, pSql->cmd.clauseIndex, index);
@ -275,6 +275,7 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
pSubQueryInfo->tsBuf = NULL; pSubQueryInfo->tsBuf = NULL;
// free result for async object will also free sqlObj // free result for async object will also free sqlObj
assert(pSubQueryInfo->exprsInfo.numOfExprs == 1); // ts_comp query only requires one resutl columns
taos_free_result(pPrevSub); taos_free_result(pPrevSub);
SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, NULL); SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, NULL);
@ -293,24 +294,26 @@ int32_t tscLaunchSecondPhaseSubqueries(SSqlObj* pSql) {
// set the second stage sub query for join process // set the second stage sub query for join process
pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_SEC_STAGE; pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_SEC_STAGE;
pQueryInfo->nAggTimeInterval = pSupporter->interval; pQueryInfo->intervalTime = pSupporter->interval;
pQueryInfo->groupbyExpr = pSupporter->groupbyExpr; pQueryInfo->groupbyExpr = pSupporter->groupbyExpr;
tscColumnBaseInfoCopy(&pQueryInfo->colList, &pSupporter->colList, 0); tscColumnBaseInfoCopy(&pQueryInfo->colList, &pSupporter->colList, 0);
tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond);
tscSqlExprCopy(&pQueryInfo->exprsInfo, &pSupporter->exprsInfo, pSupporter->uid); tscSqlExprCopy(&pQueryInfo->exprsInfo, &pSupporter->exprsInfo, pSupporter->uid, false);
tscFieldInfoCopyAll(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo); tscFieldInfoCopyAll(&pQueryInfo->fieldsInfo, &pSupporter->fieldsInfo);
pSupporter->exprsInfo.numOfExprs = 0;
pSupporter->fieldsInfo.numOfOutputCols = 0;
/* /*
* if the first column of the secondary query is not ts function, add this function. * if the first column of the secondary query is not ts function, add this function.
* Because this column is required to filter with timestamp after intersecting. * Because this column is required to filter with timestamp after intersecting.
*/ */
if (pSupporter->exprsInfo.pExprs[0].functionId != TSDB_FUNC_TS) { if (pSupporter->exprsInfo.pExprs[0]->functionId != TSDB_FUNC_TS) {
tscAddTimestampColumn(pQueryInfo, TSDB_FUNC_TS, 0); tscAddTimestampColumn(pQueryInfo, TSDB_FUNC_TS, 0);
} }
// todo refactor function name
SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); assert(pNew->numOfSubs == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);

9
src/client/src/tscLocal.c
View File

@ -251,6 +251,13 @@ static int32_t tscBuildMeterSchemaResultFields(SSqlObj *pSql, int32_t numOfCols,
tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 3, TSDB_DATA_TYPE_BINARY, "Note", noteColLength); tscFieldInfoSetValue(&pQueryInfo->fieldsInfo, 3, TSDB_DATA_TYPE_BINARY, "Note", noteColLength);
rowLen += noteColLength; rowLen += noteColLength;
//set the sqlexpr part
SColumnIndex index = {0};
pQueryInfo->fieldsInfo.pSqlExpr[0] = tscSqlExprInsert(pQueryInfo, 0, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN);
pQueryInfo->fieldsInfo.pSqlExpr[1] = tscSqlExprInsert(pQueryInfo, 1, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, typeColLength, typeColLength);
pQueryInfo->fieldsInfo.pSqlExpr[2] = tscSqlExprInsert(pQueryInfo, 2, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_INT, sizeof(int32_t), sizeof(int32_t));
pQueryInfo->fieldsInfo.pSqlExpr[3] = tscSqlExprInsert(pQueryInfo, 3, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, noteColLength, noteColLength);
return rowLen; return rowLen;
} }
@ -455,6 +462,8 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa
tscInitResObjForLocalQuery(pSql, 1, valueLength); tscInitResObjForLocalQuery(pSql, 1, valueLength);
TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0); TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, 0);
pQueryInfo->fieldsInfo.pSqlExpr[0] = pQueryInfo->exprsInfo.pExprs[0];
strncpy(pRes->data, val, pField->bytes); strncpy(pRes->data, val, pField->bytes);
} }

648
src/client/src/tscSQLParser.c
View File

File diff suppressed because it is too large Load Diff

408
src/client/src/tscSecondaryMerge.c
View File

@ -13,10 +13,11 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tscSecondaryMerge.h"
#include "os.h" #include "os.h"
#include "tlosertree.h" #include "tlosertree.h"
#include "tscSecondaryMerge.h"
#include "tscUtil.h" #include "tscUtil.h"
#include "tschemautil.h"
#include "tsclient.h" #include "tsclient.h"
#include "tutil.h" #include "tutil.h"
@ -58,40 +59,37 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu
* the fields and offset attributes in pCmd and pModel may be different due to * the fields and offset attributes in pCmd and pModel may be different due to
* merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object. * merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object.
*/ */
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) {
SQLFunctionCtx *pCtx = &pReducer->pCtx[i]; SQLFunctionCtx *pCtx = &pReducer->pCtx[i];
SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, i);
pCtx->aOutputBuf = pReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pReducer->resColModel->capacity; pCtx->aOutputBuf =
pReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pReducer->resColModel->capacity;
pCtx->order = pQueryInfo->order.order; pCtx->order = pQueryInfo->order.order;
pCtx->functionId = pQueryInfo->exprsInfo.pExprs[i].functionId; pCtx->functionId = pExpr->functionId;
// input buffer hold only one point data // input buffer hold only one point data
int16_t offset = getColumnModelOffset(pDesc->pColumnModel, i); int16_t offset = getColumnModelOffset(pDesc->pColumnModel, i);
SSchema* pSchema = getColumnModelSchema(pDesc->pColumnModel, i); SSchema *pSchema = getColumnModelSchema(pDesc->pColumnModel, i);
pCtx->aInputElemBuf = pReducer->pTempBuffer->data + offset; pCtx->aInputElemBuf = pReducer->pTempBuffer->data + offset;
// input data format comes from pModel // input data format comes from pModel
pCtx->inputType = pSchema->type; pCtx->inputType = pSchema->type;
pCtx->inputBytes = pSchema->bytes; pCtx->inputBytes = pSchema->bytes;
TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i);
// output data format yet comes from pCmd. // output data format yet comes from pCmd.
pCtx->outputBytes = pField->bytes; pCtx->outputBytes = pExpr->resBytes;
pCtx->outputType = pField->type; pCtx->outputType = pExpr->resType;
pCtx->startOffset = 0; pCtx->startOffset = 0;
pCtx->size = 1; pCtx->size = 1;
pCtx->hasNull = true; pCtx->hasNull = true;
pCtx->currentStage = SECONDARY_STAGE_MERGE; pCtx->currentStage = SECONDARY_STAGE_MERGE;
pRes->bytes[i] = pField->bytes;
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
// for top/bottom function, the output of timestamp is the first column // for top/bottom function, the output of timestamp is the first column
int32_t functionId = pExpr->functionId; int32_t functionId = pExpr->functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) {
pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf; pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf;
pCtx->param[2].i64Key = pQueryInfo->order.order; pCtx->param[2].i64Key = pQueryInfo->order.order;
@ -107,12 +105,12 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SSqlRes *pRes, SLocalReducer *pRedu
pCtx->resultInfo->superTableQ = true; pCtx->resultInfo->superTableQ = true;
} }
int16_t n = 0; int16_t n = 0;
int16_t tagLen = 0; int16_t tagLen = 0;
SQLFunctionCtx** pTagCtx = calloc(pQueryInfo->fieldsInfo.numOfOutputCols, POINTER_BYTES); SQLFunctionCtx **pTagCtx = calloc(pQueryInfo->fieldsInfo.numOfOutputCols, POINTER_BYTES);
SQLFunctionCtx* pCtx = NULL; SQLFunctionCtx *pCtx = NULL;
for(int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TS_DUMMY) { if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TS_DUMMY) {
tagLen += pExpr->resBytes; tagLen += pExpr->resBytes;
@ -218,12 +216,12 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
#ifdef _DEBUG_VIEW #ifdef _DEBUG_VIEW
printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", pDS->filePage.numOfElems); printf("load data page into mem for build loser tree: %" PRIu64 " rows\n", pDS->filePage.numOfElems);
SSrcColumnInfo colInfo[256] = {0}; SSrcColumnInfo colInfo[256] = {0};
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
tscGetSrcColumnInfo(colInfo, pQueryInfo); tscGetSrcColumnInfo(colInfo, pQueryInfo);
tColModelDisplayEx(pDesc->pColumnModel, pDS->filePage.data, pDS->filePage.numOfElems, pMemBuffer[0]->numOfElemsPerPage, tColModelDisplayEx(pDesc->pColumnModel, pDS->filePage.data, pDS->filePage.numOfElems,
colInfo); pMemBuffer[0]->numOfElemsPerPage, colInfo);
#endif #endif
if (pDS->filePage.numOfElems == 0) { // no data in this flush if (pDS->filePage.numOfElems == 0) { // no data in this flush
tscTrace("%p flush data is empty, ignore %d flush record", pSqlObjAddr, idx); tscTrace("%p flush data is empty, ignore %d flush record", pSqlObjAddr, idx);
@ -244,8 +242,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
param->pLocalData = pReducer->pLocalDataSrc; param->pLocalData = pReducer->pLocalDataSrc;
param->pDesc = pReducer->pDesc; param->pDesc = pReducer->pDesc;
param->numOfElems = pReducer->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage; param->numOfElems = pReducer->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
param->groupOrderType = pQueryInfo->groupbyExpr.orderType; param->groupOrderType = pQueryInfo->groupbyExpr.orderType;
pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator); pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator);
@ -255,7 +253,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
// the input data format follows the old format, but output in a new format. // the input data format follows the old format, but output in a new format.
// so, all the input must be parsed as old format // so, all the input must be parsed as old format
pReducer->pCtx = (SQLFunctionCtx *)calloc(pQueryInfo->fieldsInfo.numOfOutputCols, sizeof(SQLFunctionCtx)); pReducer->pCtx = (SQLFunctionCtx *)calloc(pQueryInfo->exprsInfo.numOfExprs, sizeof(SQLFunctionCtx));
pReducer->rowSize = pMemBuffer[0]->nElemSize; pReducer->rowSize = pMemBuffer[0]->nElemSize;
@ -279,7 +277,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16; pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16;
pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage)); pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage));
int32_t finalRowLength = tscGetResRowLength(pQueryInfo); int32_t finalRowLength = tscGetResRowLength(pQueryInfo);
pReducer->resColModel = finalmodel; pReducer->resColModel = finalmodel;
pReducer->resColModel->capacity = pReducer->nResultBufSize / finalRowLength; pReducer->resColModel->capacity = pReducer->nResultBufSize / finalRowLength;
@ -288,12 +286,12 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity); pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity);
pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize); pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize);
if (pReducer->pTempBuffer == NULL|| pReducer->discardData == NULL || pReducer->pResultBuf == NULL || if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL ||
pReducer->pBufForInterpo == NULL || pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { pReducer->pBufForInterpo == NULL || pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) {
tfree(pReducer->pTempBuffer); tfree(pReducer->pTempBuffer);
tfree(pReducer->discardData); tfree(pReducer->discardData);
tfree(pReducer->pResultBuf); tfree(pReducer->pResultBuf);
tfree(pReducer->pFinalRes); tfree(pReducer->pFinalRes);
tfree(pReducer->pBufForInterpo); tfree(pReducer->pBufForInterpo);
tfree(pReducer->prevRowOfInput); tfree(pReducer->prevRowOfInput);
@ -302,30 +300,31 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
} }
pReducer->pTempBuffer->numOfElems = 0; pReducer->pTempBuffer->numOfElems = 0;
pReducer->pResInfo = calloc((size_t)pQueryInfo->fieldsInfo.numOfOutputCols, sizeof(SResultInfo)); pReducer->pResInfo = calloc((size_t)pQueryInfo->exprsInfo.numOfExprs, sizeof(SResultInfo));
tscCreateResPointerInfo(pRes, pQueryInfo); tscCreateResPointerInfo(pRes, pQueryInfo);
tscInitSqlContext(pCmd, pRes, pReducer, pDesc); tscInitSqlContext(pCmd, pRes, pReducer, pDesc);
// we change the capacity of schema to denote that there is only one row in temp buffer // we change the capacity of schema to denote that there is only one row in temp buffer
pReducer->pDesc->pColumnModel->capacity = 1; pReducer->pDesc->pColumnModel->capacity = 1;
//restore the limitation value at the last stage // restore the limitation value at the last stage
if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
pQueryInfo->limit.limit = pQueryInfo->clauseLimit; pQueryInfo->limit.limit = pQueryInfo->clauseLimit;
pQueryInfo->limit.offset = pQueryInfo->prjOffset; pQueryInfo->limit.offset = pQueryInfo->prjOffset;
} }
pReducer->offset = pQueryInfo->limit.offset; pReducer->offset = pQueryInfo->limit.offset;
pRes->pLocalReducer = pReducer; pRes->pLocalReducer = pReducer;
pRes->numOfGroups = 0; pRes->numOfGroups = 0;
SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0);
int16_t prec = pMeterMetaInfo->pMeterMeta->precision; int16_t prec = pMeterMetaInfo->pMeterMeta->precision;
int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime;
int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->nAggTimeInterval, pQueryInfo->intervalTimeUnit, prec); int64_t revisedSTime =
taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, prec);
SInterpolationInfo *pInterpoInfo = &pReducer->interpolationInfo; SInterpolationInfo *pInterpoInfo = &pReducer->interpolationInfo;
taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
@ -336,7 +335,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
pInterpoInfo->pTags[0] = (char *)pInterpoInfo->pTags + POINTER_BYTES * pQueryInfo->groupbyExpr.numOfGroupCols; pInterpoInfo->pTags[0] = (char *)pInterpoInfo->pTags + POINTER_BYTES * pQueryInfo->groupbyExpr.numOfGroupCols;
for (int32_t i = 1; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { for (int32_t i = 1; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pReducer->resColModel, startIndex + i - 1); SSchema *pSchema = getColumnModelSchema(pReducer->resColModel, startIndex + i - 1);
pInterpoInfo->pTags[i] = pSchema->bytes + pInterpoInfo->pTags[i - 1]; pInterpoInfo->pTags[i] = pSchema->bytes + pInterpoInfo->pTags[i - 1];
} }
} else { } else {
@ -388,7 +387,7 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF
int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, void *data, int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, void *data,
int32_t numOfRows, int32_t orderType) { int32_t numOfRows, int32_t orderType) {
SColumnModel *pModel = pDesc->pColumnModel; SColumnModel *pModel = pDesc->pColumnModel;
if (pPage->numOfElems + numOfRows <= pModel->capacity) { if (pPage->numOfElems + numOfRows <= pModel->capacity) {
tColModelAppend(pModel, pPage, data, 0, numOfRows, numOfRows); tColModelAppend(pModel, pPage, data, 0, numOfRows, numOfRows);
return 0; return 0;
@ -445,11 +444,11 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
return; return;
} }
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd * pCmd = &pSql->cmd;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
// there is no more result, so we release all allocated resource // there is no more result, so we release all allocated resource
SLocalReducer *pLocalReducer = (SLocalReducer*)atomic_exchange_ptr(&pRes->pLocalReducer, NULL); SLocalReducer *pLocalReducer = (SLocalReducer *)atomic_exchange_ptr(&pRes->pLocalReducer, NULL);
if (pLocalReducer != NULL) { if (pLocalReducer != NULL) {
int32_t status = 0; int32_t status = 0;
while ((status = atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, while ((status = atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY,
@ -461,19 +460,18 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
taosDestoryInterpoInfo(&pLocalReducer->interpolationInfo); taosDestoryInterpoInfo(&pLocalReducer->interpolationInfo);
if (pLocalReducer->pCtx != NULL) { if (pLocalReducer->pCtx != NULL) {
for(int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i];
tVariantDestroy(&pCtx->tag); tVariantDestroy(&pCtx->tag);
if (pCtx->tagInfo.pTagCtxList != NULL) { if (pCtx->tagInfo.pTagCtxList != NULL) {
tfree(pCtx->tagInfo.pTagCtxList); tfree(pCtx->tagInfo.pTagCtxList);
} }
} }
tfree(pLocalReducer->pCtx); tfree(pLocalReducer->pCtx);
} }
tfree(pLocalReducer->prevRowOfInput); tfree(pLocalReducer->prevRowOfInput);
tfree(pLocalReducer->pTempBuffer); tfree(pLocalReducer->pTempBuffer);
@ -514,15 +512,15 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
} }
static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCmd, SColumnModel *pModel) { static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCmd, SColumnModel *pModel) {
int32_t numOfGroupByCols = 0; int32_t numOfGroupByCols = 0;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) { if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
numOfGroupByCols = pQueryInfo->groupbyExpr.numOfGroupCols; numOfGroupByCols = pQueryInfo->groupbyExpr.numOfGroupCols;
} }
// primary timestamp column is involved in final result // primary timestamp column is involved in final result
if (pQueryInfo->nAggTimeInterval != 0 || tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { if (pQueryInfo->intervalTime != 0 || tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
numOfGroupByCols++; numOfGroupByCols++;
} }
@ -539,7 +537,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
orderIdx[i] = startCols++; orderIdx[i] = startCols++;
} }
if (pQueryInfo->nAggTimeInterval != 0) { if (pQueryInfo->intervalTime != 0) {
// the first column is the timestamp, handles queries like "interval(10m) group by tags" // the first column is the timestamp, handles queries like "interval(10m) group by tags"
orderIdx[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX; orderIdx[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX;
} }
@ -556,17 +554,17 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
} }
bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage *tmpBuffer) { bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage *tmpBuffer) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
int16_t functionId = tscSqlExprGet(pQueryInfo, 0)->functionId; int16_t functionId = tscSqlExprGet(pQueryInfo, 0)->functionId;
// disable merge procedure for column projection query // disable merge procedure for column projection query
assert(functionId != TSDB_FUNC_ARITHM); assert(functionId != TSDB_FUNC_ARITHM);
if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
return true; return true;
} }
if (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_ARITHM) { if (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_ARITHM) {
return false; return false;
} }
@ -581,10 +579,10 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
if (pOrderDesc->orderIdx.pData[numOfCols - 1] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { //<= 0 if (pOrderDesc->orderIdx.pData[numOfCols - 1] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { //<= 0
// super table interval query // super table interval query
assert(pQueryInfo->nAggTimeInterval > 0); assert(pQueryInfo->intervalTime > 0);
pOrderDesc->orderIdx.numOfCols -= 1; pOrderDesc->orderIdx.numOfCols -= 1;
} else { // simple group by query } else { // simple group by query
assert(pQueryInfo->nAggTimeInterval == 0); assert(pQueryInfo->intervalTime == 0);
} }
// only one row exists // only one row exists
@ -599,11 +597,11 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
SSchema * pSchema = NULL; SSchema * pSchema = NULL;
SColumnModel *pModel = NULL; SColumnModel *pModel = NULL;
*pFinalModel = NULL; *pFinalModel = NULL;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0);
(*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pMeterMetaInfo->pMetricMeta->numOfVnodes); (*pMemBuffer) = (tExtMemBuffer **)malloc(POINTER_BYTES * pMeterMetaInfo->pMetricMeta->numOfVnodes);
@ -613,7 +611,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
return pRes->code; return pRes->code;
} }
pSchema = (SSchema *)calloc(1, sizeof(SSchema) * pQueryInfo->fieldsInfo.numOfOutputCols); pSchema = (SSchema *)calloc(1, sizeof(SSchema) * pQueryInfo->exprsInfo.numOfExprs);
if (pSchema == NULL) { if (pSchema == NULL) {
tscError("%p failed to allocate memory", pSql); tscError("%p failed to allocate memory", pSql);
pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
@ -621,7 +619,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
} }
int32_t rlen = 0; int32_t rlen = 0;
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
pSchema[i].bytes = pExpr->resBytes; pSchema[i].bytes = pExpr->resBytes;
@ -634,8 +632,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
if (rlen != 0) { if (rlen != 0) {
capacity = nBufferSizes / rlen; capacity = nBufferSizes / rlen;
} }
pModel = createColumnModel(pSchema, pQueryInfo->fieldsInfo.numOfOutputCols, capacity); pModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity);
for (int32_t i = 0; i < pMeterMetaInfo->pMetricMeta->numOfVnodes; ++i) { for (int32_t i = 0; i < pMeterMetaInfo->pMetricMeta->numOfVnodes; ++i) {
(*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
@ -647,16 +645,43 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
return pRes->code; return pRes->code;
} }
memset(pSchema, 0, sizeof(SSchema) * pQueryInfo->fieldsInfo.numOfOutputCols); // final result depends on the fields number
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { memset(pSchema, 0, sizeof(SSchema) * pQueryInfo->exprsInfo.numOfExprs);
TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
pSchema[i].type = pField->type; SSchema *p1 = tsGetColumnSchema(pMeterMetaInfo->pMeterMeta, pExpr->colInfo.colIdx);
pSchema[i].bytes = pField->bytes;
strcpy(pSchema[i].name, pField->name); int16_t inter = 0;
int16_t type = -1;
int16_t bytes = 0;
// if ((pExpr->functionId >= TSDB_FUNC_FIRST_DST && pExpr->functionId <= TSDB_FUNC_LAST_DST) ||
// (pExpr->functionId >= TSDB_FUNC_SUM && pExpr->functionId <= TSDB_FUNC_MAX) ||
// pExpr->functionId == TSDB_FUNC_LAST_ROW) {
// the final result size and type in the same as query on single table.
// so here, set the flag to be false;
int32_t functionId = pExpr->functionId;
if (functionId >= TSDB_FUNC_TS && functionId <= TSDB_FUNC_DIFF) {
type = pModel->pFields[i].field.type;
bytes = pModel->pFields[i].field.bytes;
} else {
if (functionId == TSDB_FUNC_FIRST_DST) {
functionId = TSDB_FUNC_FIRST;
} else if (functionId == TSDB_FUNC_LAST_DST) {
functionId = TSDB_FUNC_LAST;
}
getResultDataInfo(p1->type, p1->bytes, functionId, 0, &type, &bytes, &inter, 0, false);
}
pSchema[i].type = type;
pSchema[i].bytes = bytes;
strcpy(pSchema[i].name, pModel->pFields[i].field.name);
} }
*pFinalModel = createColumnModel(pSchema, pQueryInfo->fieldsInfo.numOfOutputCols, capacity); *pFinalModel = createColumnModel(pSchema, pQueryInfo->exprsInfo.numOfExprs, capacity);
tfree(pSchema); tfree(pSchema);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -746,13 +771,15 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource *
} }
} }
void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo, SInterpolationInfo *pInterpoInfo) { void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQueryInfo,
SInterpolationInfo *pInterpoInfo) {
// discard following dataset in the same group and reset the interpolation information // discard following dataset in the same group and reset the interpolation information
SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0);
int16_t prec = pMeterMetaInfo->pMeterMeta->precision; int16_t prec = pMeterMetaInfo->pMeterMeta->precision;
int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime;
int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->nAggTimeInterval, pQueryInfo->intervalTimeUnit, prec); int64_t revisedSTime =
taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, prec);
taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, taosInitInterpoInfo(pInterpoInfo, pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
pLocalReducer->rowSize); pLocalReducer->rowSize);
@ -765,24 +792,25 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo*
} }
// todo merge with following function // todo merge with following function
static void reversedCopyResultToDstBuf(SQueryInfo* pQueryInfo, SSqlRes *pRes, tFilePage *pFinalDataPage) { // static void reversedCopyResultToDstBuf(SQueryInfo* pQueryInfo, SSqlRes *pRes, tFilePage *pFinalDataPage) {
//
for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { // for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); // TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i);
//
// int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i);
// char * src = pFinalDataPage->data + (pRes->numOfRows - 1) * pField->bytes + pRes->numOfRows * offset;
// char * dst = pRes->data + pRes->numOfRows * offset;
//
// for (int32_t j = 0; j < pRes->numOfRows; ++j) {
// memcpy(dst, src, (size_t)pField->bytes);
// dst += pField->bytes;
// src -= pField->bytes;
// }
// }
//}
int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages,
char * src = pFinalDataPage->data + (pRes->numOfRows - 1) * pField->bytes + pRes->numOfRows * offset; SLocalReducer *pLocalReducer) {
char * dst = pRes->data + pRes->numOfRows * offset;
for (int32_t j = 0; j < pRes->numOfRows; ++j) {
memcpy(dst, src, (size_t)pField->bytes);
dst += pField->bytes;
src -= pField->bytes;
}
}
}
static void reversedCopyFromInterpolationToDstBuf(SQueryInfo* pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, SLocalReducer *pLocalReducer) {
assert(0); assert(0);
for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) { for (int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i);
@ -806,11 +834,11 @@ static void reversedCopyFromInterpolationToDstBuf(SQueryInfo* pQueryInfo, SSqlRe
* by "interuptHandler" function in shell * by "interuptHandler" function in shell
*/ */
static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneOutput) { static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneOutput) {
SSqlCmd * pCmd = &pSql->cmd; SSqlCmd * pCmd = &pSql->cmd;
SSqlRes * pRes = &pSql->res; SSqlRes * pRes = &pSql->res;
tFilePage *pFinalDataPage = pLocalReducer->pResultBuf; tFilePage * pFinalDataPage = pLocalReducer->pResultBuf;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
if (pRes->pLocalReducer != pLocalReducer) { if (pRes->pLocalReducer != pLocalReducer) {
/* /*
* Release the SSqlObj is called, and it is int destroying function invoked by other thread. * Release the SSqlObj is called, and it is int destroying function invoked by other thread.
@ -820,7 +848,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
assert(pRes->pLocalReducer == NULL); assert(pRes->pLocalReducer == NULL);
} }
if (pQueryInfo->nAggTimeInterval == 0 || pQueryInfo->interpoType == TSDB_INTERPO_NONE) { if (pQueryInfo->intervalTime == 0 || pQueryInfo->interpoType == TSDB_INTERPO_NONE) {
// no interval query, no interpolation // no interval query, no interpolation
pRes->data = pLocalReducer->pFinalRes; pRes->data = pLocalReducer->pFinalRes;
pRes->numOfRows = pFinalDataPage->numOfElems; pRes->numOfRows = pFinalDataPage->numOfElems;
@ -862,19 +890,14 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
} }
int32_t rowSize = tscGetResRowLength(pQueryInfo); int32_t rowSize = tscGetResRowLength(pQueryInfo);
// handle the descend order output memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * rowSize);
// if (pQueryInfo->order.order == TSQL_SO_ASC) {
memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * rowSize);
// } else {
// reversedCopyResultToDstBuf(pQueryInfo, pRes, pFinalDataPage);
// }
pFinalDataPage->numOfElems = 0; pFinalDataPage->numOfElems = 0;
return; return;
} }
int64_t *pPrimaryKeys = (int64_t *)pLocalReducer->pBufForInterpo; int64_t *pPrimaryKeys = (int64_t *)pLocalReducer->pBufForInterpo;
SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
int64_t actualETime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; int64_t actualETime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime;
@ -889,22 +912,23 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(void *)); int32_t *functions = (int32_t *)((char *)srcData + pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(void *));
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
srcData[i] = pLocalReducer->pBufForInterpo + tscFieldInfoGetOffset(pQueryInfo, i) * pInterpoInfo->numOfRawDataInRows; srcData[i] =
pLocalReducer->pBufForInterpo + tscFieldInfoGetOffset(pQueryInfo, i) * pInterpoInfo->numOfRawDataInRows;
functions[i] = tscSqlExprGet(pQueryInfo, i)->functionId; functions[i] = tscSqlExprGet(pQueryInfo, i)->functionId;
} }
SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0);
int8_t precision = pMeterMetaInfo->pMeterMeta->precision; int8_t precision = pMeterMetaInfo->pMeterMeta->precision;
while (1) { while (1) {
int32_t remains = taosNumOfRemainPoints(pInterpoInfo); int32_t remains = taosNumOfRemainPoints(pInterpoInfo);
TSKEY etime = taosGetRevisedEndKey(actualETime, pQueryInfo->order.order, pQueryInfo->nAggTimeInterval, pQueryInfo->intervalTimeUnit, TSKEY etime = taosGetRevisedEndKey(actualETime, pQueryInfo->order.order, pQueryInfo->intervalTime,
precision); pQueryInfo->intervalTimeUnit, precision);
int32_t nrows = taosGetNumOfResultWithInterpo(pInterpoInfo, pPrimaryKeys, remains, pQueryInfo->nAggTimeInterval, etime, int32_t nrows = taosGetNumOfResultWithInterpo(pInterpoInfo, pPrimaryKeys, remains, pQueryInfo->intervalTime, etime,
pLocalReducer->resColModel->capacity); pLocalReducer->resColModel->capacity);
int32_t newRows = taosDoInterpoResult(pInterpoInfo, pQueryInfo->interpoType, pResPages, remains, nrows, int32_t newRows = taosDoInterpoResult(pInterpoInfo, pQueryInfo->interpoType, pResPages, remains, nrows,
pQueryInfo->nAggTimeInterval, pPrimaryKeys, pLocalReducer->resColModel, srcData, pQueryInfo->intervalTime, pPrimaryKeys, pLocalReducer->resColModel, srcData,
pQueryInfo->defaultVal, functions, pLocalReducer->resColModel->capacity); pQueryInfo->defaultVal, functions, pLocalReducer->resColModel->capacity);
assert(newRows <= nrows); assert(newRows <= nrows);
@ -914,7 +938,8 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
if (pQueryInfo->limit.offset > 0) { if (pQueryInfo->limit.offset > 0) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i);
memmove(pResPages[i]->data, pResPages[i]->data + pField->bytes * pQueryInfo->limit.offset, newRows * pField->bytes); memmove(pResPages[i]->data, pResPages[i]->data + pField->bytes * pQueryInfo->limit.offset,
newRows * pField->bytes);
} }
} }
@ -937,7 +962,7 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
/* all output for current group are completed */ /* all output for current group are completed */
int32_t totalRemainRows = int32_t totalRemainRows =
taosGetNumOfResWithoutLimit(pInterpoInfo, pPrimaryKeys, rpoints, pQueryInfo->nAggTimeInterval, actualETime); taosGetNumOfResWithoutLimit(pInterpoInfo, pPrimaryKeys, rpoints, pQueryInfo->intervalTime, actualETime);
if (totalRemainRows <= 0) { if (totalRemainRows <= 0) {
break; break;
} }
@ -962,10 +987,10 @@ static void doInterpolateResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, boo
if (pQueryInfo->order.order == TSQL_SO_ASC) { if (pQueryInfo->order.order == TSQL_SO_ASC) {
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i);
int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i);
memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows);
} }
} else {//todo bug?? } else { // todo bug??
reversedCopyFromInterpolationToDstBuf(pQueryInfo, pRes, pResPages, pLocalReducer); reversedCopyFromInterpolationToDstBuf(pQueryInfo, pRes, pResPages, pLocalReducer);
} }
} }
@ -985,9 +1010,9 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer)
// copy to previous temp buffer // copy to previous temp buffer
for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) { for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pColumnModel, i); SSchema *pSchema = getColumnModelSchema(pColumnModel, i);
int16_t offset = getColumnModelOffset(pColumnModel, i); int16_t offset = getColumnModelOffset(pColumnModel, i);
memcpy(pLocalReducer->prevRowOfInput + offset, tmpBuffer->data + offset, pSchema->bytes); memcpy(pLocalReducer->prevRowOfInput + offset, tmpBuffer->data + offset, pSchema->bytes);
} }
@ -995,19 +1020,19 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer)
pLocalReducer->hasPrevRow = true; pLocalReducer->hasPrevRow = true;
} }
static void doExecuteSecondaryMerge(SSqlCmd* pCmd, SLocalReducer *pLocalReducer, bool needInit) { static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, bool needInit) {
// the tag columns need to be set before all functions execution // the tag columns need to be set before all functions execution
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
for(int32_t j = 0; j < pQueryInfo->fieldsInfo.numOfOutputCols; ++j) { for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) {
SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, j); SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, j);
SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j]; SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j];
tVariantAssign(&pCtx->param[0], &pExpr->param[0]); tVariantAssign(&pCtx->param[0], &pExpr->param[0]);
// tags/tags_dummy function, the tag field of SQLFunctionCtx is from the input buffer // tags/tags_dummy function, the tag field of SQLFunctionCtx is from the input buffer
if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TAG || int32_t functionId = pExpr->functionId;
pExpr->functionId == TSDB_FUNC_TS_DUMMY) { if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS_DUMMY) {
tVariantDestroy(&pCtx->tag); tVariantDestroy(&pCtx->tag);
tVariantCreateFromBinary(&pCtx->tag, pCtx->aInputElemBuf, pCtx->inputBytes, pCtx->inputType); tVariantCreateFromBinary(&pCtx->tag, pCtx->aInputElemBuf, pCtx->inputBytes, pCtx->inputType);
} }
@ -1019,7 +1044,7 @@ static void doExecuteSecondaryMerge(SSqlCmd* pCmd, SLocalReducer *pLocalReducer,
} }
} }
for (int32_t j = 0; j < pQueryInfo->fieldsInfo.numOfOutputCols; ++j) { for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) {
int32_t functionId = tscSqlExprGet(pQueryInfo, j)->functionId; int32_t functionId = tscSqlExprGet(pQueryInfo, j)->functionId;
if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
continue; continue;
@ -1029,7 +1054,7 @@ static void doExecuteSecondaryMerge(SSqlCmd* pCmd, SLocalReducer *pLocalReducer,
} }
} }
static void handleUnprocessedRow(SSqlCmd* pCmd, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { static void handleUnprocessedRow(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) {
if (pLocalReducer->hasUnprocessedRow) { if (pLocalReducer->hasUnprocessedRow) {
pLocalReducer->hasUnprocessedRow = false; pLocalReducer->hasUnprocessedRow = false;
doExecuteSecondaryMerge(pCmd, pLocalReducer, true); doExecuteSecondaryMerge(pCmd, pLocalReducer, true);
@ -1039,14 +1064,22 @@ static void handleUnprocessedRow(SSqlCmd* pCmd, SLocalReducer *pLocalReducer, tF
static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) { static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) {
int64_t maxOutput = 0; int64_t maxOutput = 0;
for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) { for (int32_t j = 0; j < pQueryInfo->exprsInfo.numOfExprs; ++j) {
int32_t functionId = tscSqlExprGet(pQueryInfo, j)->functionId; // SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[j];
// if (pExpr == NULL) {
// assert(pQueryInfo->fieldsInfo.pExpr[j] != NULL);
//
// maxOutput = 1;
// continue;
// }
/* /*
* ts, tag, tagprj function can not decide the output number of current query * ts, tag, tagprj function can not decide the output number of current query
* the number of output result is decided by main output * the number of output result is decided by main output
*/ */
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, j);
int32_t functionId = pExpr->functionId;
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) {
continue; continue;
} }
@ -1055,6 +1088,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
maxOutput = GET_RES_INFO(&pCtx[j])->numOfRes; maxOutput = GET_RES_INFO(&pCtx[j])->numOfRes;
} }
} }
return maxOutput; return maxOutput;
} }
@ -1064,9 +1098,9 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
* filled with the same result, which is the tags, specified in group by clause * filled with the same result, which is the tags, specified in group by clause
* *
*/ */
static void fillMultiRowsOfTagsVal(SQueryInfo* pQueryInfo, int32_t numOfRes, SLocalReducer *pLocalReducer) { static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLocalReducer *pLocalReducer) {
int32_t maxBufSize = 0; // find the max tags column length to prepare the buffer int32_t maxBufSize = 0; // find the max tags column length to prepare the buffer
for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k);
if (maxBufSize < pExpr->resBytes && pExpr->functionId == TSDB_FUNC_TAG) { if (maxBufSize < pExpr->resBytes && pExpr->functionId == TSDB_FUNC_TAG) {
maxBufSize = pExpr->resBytes; maxBufSize = pExpr->resBytes;
@ -1075,8 +1109,8 @@ static void fillMultiRowsOfTagsVal(SQueryInfo* pQueryInfo, int32_t numOfRes, SLo
assert(maxBufSize >= 0); assert(maxBufSize >= 0);
char *buf = malloc((size_t) maxBufSize); char *buf = malloc((size_t)maxBufSize);
for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k);
if (pExpr->functionId != TSDB_FUNC_TAG) { if (pExpr->functionId != TSDB_FUNC_TAG) {
continue; continue;
@ -1097,13 +1131,10 @@ static void fillMultiRowsOfTagsVal(SQueryInfo* pQueryInfo, int32_t numOfRes, SLo
free(buf); free(buf);
} }
int32_t finalizeRes(SQueryInfo* pQueryInfo, SLocalReducer *pLocalReducer) { int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {
for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k);
aAggs[pExpr->functionId].xFinalize(&pLocalReducer->pCtx[k]); aAggs[pExpr->functionId].xFinalize(&pLocalReducer->pCtx[k]);
// allow to re-initialize for the next round
pLocalReducer->pCtx[k].resultInfo->initialized = false;
} }
pLocalReducer->hasPrevRow = false; pLocalReducer->hasPrevRow = false;
@ -1122,7 +1153,7 @@ int32_t finalizeRes(SQueryInfo* pQueryInfo, SLocalReducer *pLocalReducer) {
* results generated by simple aggregation function, we merge them all into one points * results generated by simple aggregation function, we merge them all into one points
* *Exception*: column projection query, required no merge procedure * *Exception*: column projection query, required no merge procedure
*/ */
bool needToMerge(SQueryInfo* pQueryInfo, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { bool needToMerge(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) {
int32_t ret = 0; // merge all result by default int32_t ret = 0; // merge all result by default
int16_t functionId = tscSqlExprGet(pQueryInfo, 0)->functionId; int16_t functionId = tscSqlExprGet(pQueryInfo, 0)->functionId;
@ -1144,7 +1175,7 @@ bool needToMerge(SQueryInfo* pQueryInfo, SLocalReducer *pLocalReducer, tFilePage
return (ret == 0); return (ret == 0);
} }
static bool reachGroupResultLimit(SQueryInfo* pQueryInfo, SSqlRes *pRes) { static bool reachGroupResultLimit(SQueryInfo *pQueryInfo, SSqlRes *pRes) {
return (pRes->numOfGroups >= pQueryInfo->slimit.limit && pQueryInfo->slimit.limit >= 0); return (pRes->numOfGroups >= pQueryInfo->slimit.limit && pQueryInfo->slimit.limit >= 0);
} }
@ -1152,7 +1183,7 @@ static bool saveGroupResultInfo(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
pRes->numOfGroups += 1; pRes->numOfGroups += 1;
// the output group is limited by the slimit clause // the output group is limited by the slimit clause
@ -1175,11 +1206,11 @@ static bool saveGroupResultInfo(SSqlObj *pSql) {
* @return if current group is skipped, return false, and do NOT record it into pRes->numOfGroups * @return if current group is skipped, return false, and do NOT record it into pRes->numOfGroups
*/ */
bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCurrentGroupRes) { bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCurrentGroupRes) {
SSqlCmd * pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes * pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
tFilePage *pResBuf = pLocalReducer->pResultBuf; tFilePage * pResBuf = pLocalReducer->pResultBuf;
SColumnModel *pModel = pLocalReducer->resColModel; SColumnModel *pModel = pLocalReducer->resColModel;
pRes->code = TSDB_CODE_SUCCESS; pRes->code = TSDB_CODE_SUCCESS;
@ -1207,11 +1238,10 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no
int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols; int32_t startIndex = pQueryInfo->fieldsInfo.numOfOutputCols - pQueryInfo->groupbyExpr.numOfGroupCols;
for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) { for (int32_t i = 0; i < pQueryInfo->groupbyExpr.numOfGroupCols; ++i) {
int16_t offset = getColumnModelOffset(pModel, startIndex + i); int16_t offset = getColumnModelOffset(pModel, startIndex + i);
SSchema* pSchema = getColumnModelSchema(pModel, startIndex + i); SSchema *pSchema = getColumnModelSchema(pModel, startIndex + i);
memcpy(pInterpoInfo->pTags[i], memcpy(pInterpoInfo->pTags[i], pLocalReducer->pBufForInterpo + offset * pResBuf->numOfElems, pSchema->bytes);
pLocalReducer->pBufForInterpo + offset * pResBuf->numOfElems, pSchema->bytes);
} }
taosInterpoSetStartInfo(&pLocalReducer->interpolationInfo, pResBuf->numOfElems, pQueryInfo->interpoType); taosInterpoSetStartInfo(&pLocalReducer->interpolationInfo, pResBuf->numOfElems, pQueryInfo->interpoType);
@ -1220,7 +1250,7 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no
return true; return true;
} }
void resetOutputBuf(SQueryInfo* pQueryInfo, SLocalReducer *pLocalReducer) { // reset output buffer to the beginning void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // reset output buffer to the beginning
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pCtx[i].aOutputBuf =
pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity; pLocalReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pLocalReducer->resColModel->capacity;
@ -1233,21 +1263,22 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer
// In handling data in other groups, we need to reset the interpolation information for a new group data // In handling data in other groups, we need to reset the interpolation information for a new group data
pRes->numOfRows = 0; pRes->numOfRows = 0;
pRes->numOfTotalInCurrentClause = 0; pRes->numOfTotalInCurrentClause = 0;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
pQueryInfo->limit.offset = pLocalReducer->offset; pQueryInfo->limit.offset = pLocalReducer->offset;
SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0);
int16_t precision = pMeterMetaInfo->pMeterMeta->precision; int16_t precision = pMeterMetaInfo->pMeterMeta->precision;
// for group result interpolation, do not return if not data is generated // for group result interpolation, do not return if not data is generated
if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime; int64_t stime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->stime : pQueryInfo->etime;
int64_t newTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->nAggTimeInterval, pQueryInfo->intervalTimeUnit, precision); int64_t newTime =
taosGetIntervalStartTimestamp(stime, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, precision);
taosInitInterpoInfo(&pLocalReducer->interpolationInfo, pQueryInfo->order.order, newTime, pQueryInfo->groupbyExpr.numOfGroupCols, taosInitInterpoInfo(&pLocalReducer->interpolationInfo, pQueryInfo->order.order, newTime,
pLocalReducer->rowSize); pQueryInfo->groupbyExpr.numOfGroupCols, pLocalReducer->rowSize);
} }
} }
@ -1259,12 +1290,12 @@ static bool doInterpolationForCurrentGroup(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
SLocalReducer * pLocalReducer = pRes->pLocalReducer; SLocalReducer * pLocalReducer = pRes->pLocalReducer;
SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo; SInterpolationInfo *pInterpoInfo = &pLocalReducer->interpolationInfo;
SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0);
int8_t p = pMeterMetaInfo->pMeterMeta->precision; int8_t p = pMeterMetaInfo->pMeterMeta->precision;
if (taosHasRemainsDataForInterpolation(pInterpoInfo)) { if (taosHasRemainsDataForInterpolation(pInterpoInfo)) {
assert(pQueryInfo->interpoType != TSDB_INTERPO_NONE); assert(pQueryInfo->interpoType != TSDB_INTERPO_NONE);
@ -1273,9 +1304,10 @@ static bool doInterpolationForCurrentGroup(SSqlObj *pSql) {
int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pInterpoInfo->numOfRawDataInRows - 1)); int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pInterpoInfo->numOfRawDataInRows - 1));
int32_t remain = taosNumOfRemainPoints(pInterpoInfo); int32_t remain = taosNumOfRemainPoints(pInterpoInfo);
TSKEY ekey = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->nAggTimeInterval, pQueryInfo->intervalTimeUnit, p); TSKEY ekey =
taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime, pQueryInfo->intervalTimeUnit, p);
int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pLocalReducer->pBufForInterpo, remain, int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, (TSKEY *)pLocalReducer->pBufForInterpo, remain,
pQueryInfo->nAggTimeInterval, ekey, pLocalReducer->resColModel->capacity); pQueryInfo->intervalTime, ekey, pLocalReducer->resColModel->capacity);
if (rows > 0) { // do interpo if (rows > 0) { // do interpo
doInterpolateResult(pSql, pLocalReducer, false); doInterpolateResult(pSql, pLocalReducer, false);
} }
@ -1295,9 +1327,9 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow; bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0);
int8_t precision = pMeterMetaInfo->pMeterMeta->precision; int8_t precision = pMeterMetaInfo->pMeterMeta->precision;
if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL || if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL ||
prevGroupCompleted) { prevGroupCompleted) {
@ -1305,8 +1337,9 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) { if (pQueryInfo->interpoType != TSDB_INTERPO_NONE) {
int64_t etime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime; int64_t etime = (pQueryInfo->stime < pQueryInfo->etime) ? pQueryInfo->etime : pQueryInfo->stime;
etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->nAggTimeInterval, pQueryInfo->intervalTimeUnit, precision); etime = taosGetRevisedEndKey(etime, pQueryInfo->order.order, pQueryInfo->intervalTime,
int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, NULL, 0, pQueryInfo->nAggTimeInterval, etime, pQueryInfo->intervalTimeUnit, precision);
int32_t rows = taosGetNumOfResultWithInterpo(pInterpoInfo, NULL, 0, pQueryInfo->intervalTime, etime,
pLocalReducer->resColModel->capacity); pLocalReducer->resColModel->capacity);
if (rows > 0) { // do interpo if (rows > 0) { // do interpo
doInterpolateResult(pSql, pLocalReducer, true); doInterpolateResult(pSql, pLocalReducer, true);
@ -1334,15 +1367,15 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
return false; return false;
} }
static void doMergeWithPrevRows(SSqlObj *pSql, int32_t numOfRes) { static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) {
SSqlCmd * pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes * pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
SLocalReducer *pLocalReducer = pRes->pLocalReducer; SLocalReducer *pLocalReducer = pRes->pLocalReducer;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
for (int32_t k = 0; k < pQueryInfo->fieldsInfo.numOfOutputCols; ++k) { for (int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, k); SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, k);
SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k];
pCtx->aOutputBuf += pCtx->outputBytes * numOfRes; pCtx->aOutputBuf += pCtx->outputBytes * numOfRes;
@ -1359,24 +1392,24 @@ static void doMergeWithPrevRows(SSqlObj *pSql, int32_t numOfRes) {
int32_t tscDoLocalreduce(SSqlObj *pSql) { int32_t tscDoLocalreduce(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
tscResetForNextRetrieve(pRes); tscResetForNextRetrieve(pRes);
if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed
tscTrace("%s call the drop local reducer", __FUNCTION__); tscTrace("%s call the drop local reducer", __FUNCTION__);
tscDestroyLocalReducer(pSql); tscDestroyLocalReducer(pSql);
return 0; return 0;
} }
SLocalReducer *pLocalReducer = pRes->pLocalReducer; SLocalReducer *pLocalReducer = pRes->pLocalReducer;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
// set the data merge in progress // set the data merge in progress
int32_t prevStatus = int32_t prevStatus =
atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS); atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS);
if (prevStatus != TSC_LOCALREDUCE_READY || pLocalReducer == NULL) { if (prevStatus != TSC_LOCALREDUCE_READY || pLocalReducer == NULL) {
assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); // it is in tscDestroyLocalReducer function already assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); // it is in tscDestroyLocalReducer function already
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1474,8 +1507,7 @@ int32_t tscDoLocalreduce(SSqlObj *pSql) {
* if the previous group does NOT generate any result (pResBuf->numOfElems == 0), * if the previous group does NOT generate any result (pResBuf->numOfElems == 0),
* continue to process results instead of return results. * continue to process results instead of return results.
*/ */
if ((!sameGroup && pResBuf->numOfElems > 0) || if ((!sameGroup && pResBuf->numOfElems > 0) || (pResBuf->numOfElems == pLocalReducer->resColModel->capacity)) {
(pResBuf->numOfElems == pLocalReducer->resColModel->capacity)) {
// does not belong to the same group // does not belong to the same group
bool notSkipped = doGenerateFinalResults(pSql, pLocalReducer, !sameGroup); bool notSkipped = doGenerateFinalResults(pSql, pLocalReducer, !sameGroup);
@ -1528,7 +1560,7 @@ int32_t tscDoLocalreduce(SSqlObj *pSql) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} else { // result buffer is not full } else { // result buffer is not full
doMergeWithPrevRows(pSql, numOfRes); doProcessResultInNextWindow(pSql, numOfRes);
savePreviousRow(pLocalReducer, tmpBuffer); savePreviousRow(pLocalReducer, tmpBuffer);
} }
} }

72
src/client/src/tscServer.c
View File

@ -231,7 +231,7 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) {
SSqlCmd * pCmd = &pSql->cmd; SSqlCmd * pCmd = &pSql->cmd;
SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0); SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, pCmd->clauseIndex, 0);
if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { // multiple vnode query if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { // multiple vnode query
SVnodeSidList *vnodeList = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, pMeterMetaInfo->vnodeIndex); SVnodeSidList *vnodeList = tscGetVnodeSidList(pMeterMetaInfo->pMetricMeta, pMeterMetaInfo->vnodeIndex);
if (vnodeList != NULL) { if (vnodeList != NULL) {
@ -254,6 +254,8 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) {
while (pSql->retry < pSql->maxRetry) { while (pSql->retry < pSql->maxRetry) {
(pSql->retry)++; (pSql->retry)++;
pSql->index = pSql->index%TSDB_VNODES_SUPPORT;
char ipstr[40] = {0}; char ipstr[40] = {0};
if (pVPeersDesc[pSql->index].ip == 0) { if (pVPeersDesc[pSql->index].ip == 0) {
/* /*
@ -643,19 +645,23 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu
tscColumnBaseInfoUpdateTableIndex(&pNewQueryInfo->colList, 0); tscColumnBaseInfoUpdateTableIndex(&pNewQueryInfo->colList, 0);
tscColumnBaseInfoCopy(&pSupporter->colList, &pNewQueryInfo->colList, 0); tscColumnBaseInfoCopy(&pSupporter->colList, &pNewQueryInfo->colList, 0);
tscSqlExprCopy(&pSupporter->exprsInfo, &pNewQueryInfo->exprsInfo, pSupporter->uid); tscSqlExprCopy(&pSupporter->exprsInfo, &pNewQueryInfo->exprsInfo, pSupporter->uid, false);
tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo); tscFieldInfoCopyAll(&pSupporter->fieldsInfo, &pNewQueryInfo->fieldsInfo);
tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond); tscTagCondCopy(&pSupporter->tagCond, &pNewQueryInfo->tagCond);
pNew->cmd.numOfCols = 0; pNew->cmd.numOfCols = 0;
pNewQueryInfo->nAggTimeInterval = 0; pNewQueryInfo->intervalTime = 0;
memset(&pNewQueryInfo->limit, 0, sizeof(SLimitVal)); memset(&pNewQueryInfo->limit, 0, sizeof(SLimitVal));
// backup the data and clear it in the sqlcmd object // backup the data and clear it in the sqlcmd object
pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr; pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr;
memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr));
// this data needs to be transfer to support struct
pNewQueryInfo->fieldsInfo.numOfOutputCols = 0;
pNewQueryInfo->exprsInfo.numOfExprs = 0;
// set the ts,tags that involved in join, as the output column of intermediate result // set the ts,tags that involved in join, as the output column of intermediate result
tscClearSubqueryInfo(&pNew->cmd); tscClearSubqueryInfo(&pNew->cmd);
@ -901,7 +907,7 @@ int tscLaunchSTableSubqueries(SSqlObj *pSql) {
tExtMemBuffer ** pMemoryBuf = NULL; tExtMemBuffer ** pMemoryBuf = NULL;
tOrderDescriptor *pDesc = NULL; tOrderDescriptor *pDesc = NULL;
SColumnModel * pModel = NULL; SColumnModel * pModel = NULL;
pRes->qhandle = 1; // hack the qhandle check pRes->qhandle = 1; // hack the qhandle check
@ -1318,8 +1324,6 @@ void tscKillMetricQuery(SSqlObj *pSql) {
taosStopRpcConn(pSql->pSubs[i]->thandle); taosStopRpcConn(pSql->pSubs[i]->thandle);
} }
pSql->numOfSubs = 0;
/* /*
* 1. if the subqueries are not launched or partially launched, we need to waiting the launched * 1. if the subqueries are not launched or partially launched, we need to waiting the launched
* query return to successfully free allocated resources. * query return to successfully free allocated resources.
@ -1537,7 +1541,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
int32_t srcColListSize = pQueryInfo->colList.numOfCols * sizeof(SColumnInfo); int32_t srcColListSize = pQueryInfo->colList.numOfCols * sizeof(SColumnInfo);
int32_t exprSize = sizeof(SSqlFuncExprMsg) * pQueryInfo->fieldsInfo.numOfOutputCols; int32_t exprSize = sizeof(SSqlFuncExprMsg) * pQueryInfo->exprsInfo.numOfExprs;
SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0);
// meter query without tags values // meter query without tags values
@ -1546,11 +1550,10 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
} }
SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta;
SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pMeterMetaInfo->vnodeIndex); SVnodeSidList *pVnodeSidList = tscGetVnodeSidList(pMetricMeta, pMeterMetaInfo->vnodeIndex);
int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(SMeterSidExtInfo)) * pVnodeSidList->numOfSids; int32_t meterInfoSize = (pMetricMeta->tagLen + sizeof(SMeterSidExtInfo)) * pVnodeSidList->numOfSids;
int32_t outputColumnSize = pQueryInfo->fieldsInfo.numOfOutputCols * sizeof(SSqlFuncExprMsg); int32_t outputColumnSize = pQueryInfo->exprsInfo.numOfExprs * sizeof(SSqlFuncExprMsg);
int32_t size = meterInfoSize + outputColumnSize + srcColListSize + exprSize + MIN_QUERY_MSG_PKT_SIZE; int32_t size = meterInfoSize + outputColumnSize + srcColListSize + exprSize + MIN_QUERY_MSG_PKT_SIZE;
if (pQueryInfo->tsBuf != NULL) { if (pQueryInfo->tsBuf != NULL) {
@ -1566,7 +1569,7 @@ static char *doSerializeTableInfo(SSqlObj *pSql, int32_t numOfMeters, int32_t vn
SMeterMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta; SMeterMeta * pMeterMeta = pMeterMetaInfo->pMeterMeta;
SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta;
tscTrace("%p vid:%d, query on %d meters", pSql, htons(vnodeId), numOfMeters); tscTrace("%p vid:%d, query on %d meters", pSql, vnodeId, numOfMeters);
if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) { if (UTIL_METER_IS_NOMRAL_METER(pMeterMetaInfo)) {
#ifdef _DEBUG_VIEW #ifdef _DEBUG_VIEW
tscTrace("%p sid:%d, uid:%" PRIu64, pSql, pMeterMetaInfo->pMeterMeta->sid, pMeterMetaInfo->pMeterMeta->uid); tscTrace("%p sid:%d, uid:%" PRIu64, pSql, pMeterMetaInfo->pMeterMeta->sid, pMeterMetaInfo->pMeterMeta->uid);
@ -1683,12 +1686,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return -1; return -1;
} }
pQueryMsg->nAggTimeInterval = htobe64(pQueryInfo->nAggTimeInterval); pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime);
pQueryMsg->intervalTimeUnit = pQueryInfo->intervalTimeUnit; pQueryMsg->intervalTimeUnit = pQueryInfo->intervalTimeUnit;
pQueryMsg->slidingTime = htobe64(pQueryInfo->nSlidingTime); pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime);
if (pQueryInfo->nAggTimeInterval < 0) { if (pQueryInfo->intervalTime < 0) {
tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->nAggTimeInterval); tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime);
return -1; return -1;
} }
@ -1768,7 +1771,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg; SSqlFuncExprMsg *pSqlFuncExpr = (SSqlFuncExprMsg *)pMsg;
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) {
SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i);
if (pExpr->functionId == TSDB_FUNC_ARITHM) { if (pExpr->functionId == TSDB_FUNC_ARITHM) {
@ -1862,6 +1865,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
*((int16_t *)pMsg) += pCol->flag; *((int16_t *)pMsg) += pCol->flag;
pMsg += sizeof(pCol->flag); pMsg += sizeof(pCol->flag);
memcpy(pMsg, pCol->name, tListLen(pCol->name));
pMsg += tListLen(pCol->name);
} }
} }
@ -2491,16 +2497,8 @@ static int tscSetResultPointer(SQueryInfo *pQueryInfo, SSqlRes *pRes) {
} }
for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i);
int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); pRes->tsrow[i] = (pRes->data + offset * pRes->numOfRows);
pRes->bytes[i] = pField->bytes;
// if (pQueryInfo->order.order == TSQL_SO_DESC) {
// pRes->bytes[i] = -pRes->bytes[i];
// pRes->tsrow[i] = ((pRes->data + offset * pRes->numOfRows) + (pRes->numOfRows - 1) * pField->bytes);
// } else {
pRes->tsrow[i] = (pRes->data + offset * pRes->numOfRows);
// }
} }
return 0; return 0;
@ -2725,8 +2723,10 @@ static int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) {
int32_t joinCondLen = (TSDB_METER_ID_LEN + sizeof(int16_t)) * 2; int32_t joinCondLen = (TSDB_METER_ID_LEN + sizeof(int16_t)) * 2;
int32_t elemSize = sizeof(SMetricMetaElemMsg) * pQueryInfo->numOfTables; int32_t elemSize = sizeof(SMetricMetaElemMsg) * pQueryInfo->numOfTables;
int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndexEx);
int32_t len = tagLen + joinCondLen + elemSize + defaultSize; int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize;
return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE);
} }
@ -2795,7 +2795,7 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
int32_t condLen = 0; int32_t condLen = 0;
if (pTagCond->numOfTagCond > 0) { if (pTagCond->numOfTagCond > 0) {
SCond *pCond = tsGetMetricQueryCondPos(pTagCond, uid); SCond *pCond = tsGetMetricQueryCondPos(pTagCond, uid);
if (pCond != NULL) { if (pCond != NULL && pCond->cond != NULL) {
condLen = strlen(pCond->cond) + 1; condLen = strlen(pCond->cond) + 1;
bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE); bool ret = taosMbsToUcs4(pCond->cond, condLen, pMsg, condLen * TSDB_NCHAR_SIZE);
@ -2817,11 +2817,14 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
offset = pMsg - (char *)pMetaMsg; offset = pMsg - (char *)pMetaMsg;
pElem->tableCond = htonl(offset); pElem->tableCond = htonl(offset);
uint32_t len = strlen(pTagCond->tbnameCond.cond); uint32_t len = 0;
if (pTagCond->tbnameCond.cond != NULL) {
len = strlen(pTagCond->tbnameCond.cond);
memcpy(pMsg, pTagCond->tbnameCond.cond, len);
}
pElem->tableCondLen = htonl(len); pElem->tableCondLen = htonl(len);
memcpy(pMsg, pTagCond->tbnameCond.cond, len);
pMsg += len; pMsg += len;
} }
@ -2851,6 +2854,7 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pDestCol->colIdx = htons(pCol->colIdx); pDestCol->colIdx = htons(pCol->colIdx);
pDestCol->colId = htons(pDestCol->colId); pDestCol->colId = htons(pDestCol->colId);
pDestCol->flag = htons(pDestCol->flag); pDestCol->flag = htons(pDestCol->flag);
strncpy(pDestCol->name, pCol->name, tListLen(pCol->name));
pMsg += sizeof(SColIndexEx); pMsg += sizeof(SColIndexEx);
} }
@ -3288,6 +3292,7 @@ int tscProcessShowRsp(SSqlObj *pSql) {
int32_t size = pMeta->numOfColumns * sizeof(SSchema) + sizeof(SMeterMeta); int32_t size = pMeta->numOfColumns * sizeof(SSchema) + sizeof(SMeterMeta);
pMeterMetaInfo->pMeterMeta = pMeterMetaInfo->pMeterMeta =
(SMeterMeta *)taosAddDataIntoCache(tscCacheHandle, key, (char *)pMeta, size, tsMeterMetaKeepTimer); (SMeterMeta *)taosAddDataIntoCache(tscCacheHandle, key, (char *)pMeta, size, tsMeterMetaKeepTimer);
pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols; pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutputCols;
SSchema *pMeterSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); SSchema *pMeterSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta);
@ -3298,6 +3303,9 @@ int tscProcessShowRsp(SSqlObj *pSql) {
index.columnIndex = i; index.columnIndex = i;
tscColumnBaseInfoInsert(pQueryInfo, &index); tscColumnBaseInfoInsert(pQueryInfo, &index);
tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pMeterSchema[i]); tscFieldInfoSetValFromSchema(&pQueryInfo->fieldsInfo, i, &pMeterSchema[i]);
pQueryInfo->fieldsInfo.pSqlExpr[i] = tscSqlExprInsert(pQueryInfo, i, TSDB_FUNC_TS_DUMMY, &index,
pMeterSchema[i].type, pMeterSchema[i].bytes, pMeterSchema[i].bytes);
} }
tscFieldInfoCalOffset(pQueryInfo); tscFieldInfoCalOffset(pQueryInfo);

179
src/client/src/tscSql.c
View File

@ -13,8 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h" #include <tast.h>
#include "hash.h" #include "hash.h"
#include "os.h"
#include "tcache.h" #include "tcache.h"
#include "tlog.h" #include "tlog.h"
#include "tnote.h" #include "tnote.h"
@ -200,7 +201,7 @@ int taos_query_imp(STscObj *pObj, SSqlObj *pSql) {
taosCleanUpHashTable(pSql->pTableHashList); taosCleanUpHashTable(pSql->pTableHashList);
pSql->pTableHashList = NULL; pSql->pTableHashList = NULL;
} }
tscDump("%p pObj:%p, SQL: %s", pSql, pObj, pSql->sqlstr); tscDump("%p pObj:%p, SQL: %s", pSql, pObj, pSql->sqlstr);
pRes->code = (uint8_t)tsParseSql(pSql, false); pRes->code = (uint8_t)tsParseSql(pSql, false);
@ -367,9 +368,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) {
// pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i, pQueryInfo->order) + pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i);
// pRes->bytes[i] * (1 - pQueryInfo->order.order) * (pRes->numOfRows - 1);
pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i, pQueryInfo->order);
} }
*rows = pRes->tsrow; *rows = pRes->tsrow;
@ -377,54 +376,115 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) {
return (pQueryInfo->order.order == TSQL_SO_DESC) ? pRes->numOfRows : -pRes->numOfRows; return (pQueryInfo->order.order == TSQL_SO_DESC) ? pRes->numOfRows : -pRes->numOfRows;
} }
static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) {
SSqlRes *pRes = &pSql->res;
if (isNull(pRes->tsrow[columnIndex], pField->type)) {
pRes->tsrow[columnIndex] = NULL;
} else if (pField->type == TSDB_DATA_TYPE_NCHAR) {
// convert unicode to native code in a temporary buffer extra one byte for terminated symbol
if (pRes->buffer[columnIndex] == NULL) {
pRes->buffer[columnIndex] = malloc(pField->bytes + TSDB_NCHAR_SIZE);
}
/* string terminated char for binary data*/
memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE);
if (taosUcs4ToMbs(pRes->tsrow[columnIndex], pField->bytes, pRes->buffer[columnIndex])) {
pRes->tsrow[columnIndex] = pRes->buffer[columnIndex];
} else {
tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow);
pRes->tsrow[columnIndex] = NULL;
}
}
}
static char *getArithemicInputSrc(void *param, char *name, int32_t colId) {
SArithmeticSupport *pSupport = (SArithmeticSupport *)param;
SSqlFunctionExpr * pExpr = pSupport->pExpr;
int32_t index = -1;
for (int32_t i = 0; i < pExpr->pBinExprInfo.numOfCols; ++i) {
if (strcmp(name, pExpr->pBinExprInfo.pReqColumns[i].name) == 0) {
index = i;
break;
}
}
assert(index >= 0 && index < pExpr->pBinExprInfo.numOfCols);
return pSupport->data[index] + pSupport->offset * pSupport->elemSize[index];
}
static void **doSetResultRowData(SSqlObj *pSql) { static void **doSetResultRowData(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd; SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res; SSqlRes *pRes = &pSql->res;
assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows);
if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker
tfree(pRes->tsrow); tfree(pRes->tsrow);
return pRes->tsrow; return pRes->tsrow;
} }
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
//todo refactor move away
for(int32_t k = 0; k < pQueryInfo->exprsInfo.numOfExprs; ++k) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, k);
if (k > 0) {
SSqlExpr* pPrev = tscSqlExprGet(pQueryInfo, k - 1);
pExpr->offset = pPrev->offset + pPrev->resBytes;
}
}
int32_t num = 0; int32_t num = 0;
for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutputCols; ++i) { for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) {
pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i, pQueryInfo->order) + pRes->bytes[i] * pRes->row; if (pQueryInfo->fieldsInfo.pSqlExpr[i] != NULL) {
SSqlExpr* pExpr = pQueryInfo->fieldsInfo.pSqlExpr[i];
pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pExpr->resBytes * pRes->row;
} else {
assert(0);
}
// primary key column cannot be null in interval query, no need to check // primary key column cannot be null in interval query, no need to check
if (i == 0 && pQueryInfo->nAggTimeInterval > 0) { if (i == 0 && pQueryInfo->intervalTime > 0) {
continue; continue;
} }
TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i); TAOS_FIELD *pField = tscFieldInfoGetField(pQueryInfo, i);
if (isNull(pRes->tsrow[i], pField->type)) { transferNcharData(pSql, i, pField);
pRes->tsrow[i] = NULL;
} else if (pField->type == TSDB_DATA_TYPE_NCHAR) {
// convert unicode to native code in a temporary buffer extra one byte for terminated symbol
if (pRes->buffer[num] == NULL) {
pRes->buffer[num] = malloc(pField->bytes + TSDB_NCHAR_SIZE);
}
/* string terminated char for binary data*/ // calculate the result from serveral other columns
memset(pRes->buffer[num], 0, pField->bytes + TSDB_NCHAR_SIZE); if (pQueryInfo->fieldsInfo.pExpr != NULL && pQueryInfo->fieldsInfo.pExpr[i] != NULL) {
SArithmeticSupport *sas = (SArithmeticSupport *)calloc(1, sizeof(SArithmeticSupport));
if (taosUcs4ToMbs(pRes->tsrow[i], pField->bytes, pRes->buffer[num])) { sas->offset = 0;
pRes->tsrow[i] = pRes->buffer[num]; sas->pExpr = pQueryInfo->fieldsInfo.pExpr[i];
} else {
tscError("%p charset:%s to %s. val:%ls convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, pRes->tsrow); sas->numOfCols = sas->pExpr->pBinExprInfo.numOfCols;
pRes->tsrow[i] = NULL;
if (pRes->buffer[i] == NULL) {
pRes->buffer[i] = malloc(tscFieldInfoGetField(pQueryInfo, i)->bytes);
} }
num++; for(int32_t k = 0; k < sas->numOfCols; ++k) {
int32_t columnIndex = sas->pExpr->pBinExprInfo.pReqColumns[k].colIdxInBuf;
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, columnIndex);
sas->elemSize[k] = pExpr->resBytes;
sas->data[k] = (pRes->data + pRes->numOfRows* pExpr->offset) + pRes->row*pExpr->resBytes;
}
tSQLBinaryExprCalcTraverse(sas->pExpr->pBinExprInfo.pBinExpr, 1, pRes->buffer[i], sas, TSQL_SO_ASC, getArithemicInputSrc);
pRes->tsrow[i] = pRes->buffer[i];
free(sas); //todo optimization
} }
} }
assert(num <= pQueryInfo->fieldsInfo.numOfOutputCols); assert(num <= pQueryInfo->fieldsInfo.numOfOutputCols);
pRes->row++; // index increase one-step pRes->row++; // index increase one-step
return pRes->tsrow; return pRes->tsrow;
} }
@ -466,7 +526,7 @@ static bool tscHashRemainDataInSubqueryResultSet(SSqlObj *pSql) {
if (pSql->pSubs[i] == 0) { if (pSql->pSubs[i] == 0) {
continue; continue;
} }
SSqlRes * pRes1 = &pSql->pSubs[i]->res; SSqlRes * pRes1 = &pSql->pSubs[i]->res;
SQueryInfo *pQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[i]->cmd, 0); SQueryInfo *pQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[i]->cmd, 0);
@ -502,11 +562,10 @@ static void **tscBuildResFromSubqueries(SSqlObj *pSql) {
if (numOfTableHasRes >= 2) { // do merge result if (numOfTableHasRes >= 2) { // do merge result
success = (doSetResultRowData(pSql->pSubs[0]) != NULL) && success = (doSetResultRowData(pSql->pSubs[0]) != NULL) && (doSetResultRowData(pSql->pSubs[1]) != NULL);
(doSetResultRowData(pSql->pSubs[1]) != NULL); // TSKEY key1 = *(TSKEY *)pRes1->tsrow[0];
// TSKEY key1 = *(TSKEY *)pRes1->tsrow[0]; // TSKEY key2 = *(TSKEY *)pRes2->tsrow[0];
// TSKEY key2 = *(TSKEY *)pRes2->tsrow[0]; // printf("first:%" PRId64 ", second:%" PRId64 "\n", key1, key2);
// printf("first:%" PRId64 ", second:%" PRId64 "\n", key1, key2);
} else { // only one subquery } else { // only one subquery
SSqlObj *pSub = pSql->pSubs[0]; SSqlObj *pSub = pSql->pSubs[0];
if (pSub == NULL) { if (pSub == NULL) {
@ -596,7 +655,7 @@ TAOS_ROW taos_fetch_row_impl(TAOS_RES *res) {
tscProcessSql(pSql); // retrieve data from virtual node tscProcessSql(pSql); // retrieve data from virtual node
//if failed to retrieve data from current virtual node, try next one if exists // if failed to retrieve data from current virtual node, try next one if exists
if (hasMoreVnodesToTry(pSql)) { if (hasMoreVnodesToTry(pSql)) {
tscTryQueryNextVnode(pSql, NULL); tscTryQueryNextVnode(pSql, NULL);
} }
@ -638,7 +697,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
// current subclause is completed, try the next subclause // current subclause is completed, try the next subclause
while (rows == NULL && pCmd->clauseIndex < pCmd->numOfClause - 1) { while (rows == NULL && pCmd->clauseIndex < pCmd->numOfClause - 1) {
tscTryQueryNextClause(pSql, NULL); tscTryQueryNextClause(pSql, NULL);
// if the rows is not NULL, return immediately // if the rows is not NULL, return immediately
rows = taos_fetch_row_impl(res); rows = taos_fetch_row_impl(res);
} }
@ -701,7 +760,7 @@ int taos_select_db(TAOS *taos, const char *db) {
return taos_query(taos, sql); return taos_query(taos, sql);
} }
void taos_free_result_imp(TAOS_RES* res, int keepCmd) { void taos_free_result_imp(TAOS_RES *res, int keepCmd) {
if (res == NULL) return; if (res == NULL) return;
SSqlObj *pSql = (SSqlObj *)res; SSqlObj *pSql = (SSqlObj *)res;
@ -755,7 +814,7 @@ void taos_free_result_imp(TAOS_RES* res, int keepCmd) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
tscTrace("%p code:%d, numOfRows:%d, command:%d", pSql, pRes->code, pRes->numOfRows, pCmd->command); tscTrace("%p code:%d, numOfRows:%d, command:%d", pSql, pRes->code, pRes->numOfRows, pCmd->command);
void *fp = pSql->fp; void *fp = pSql->fp;
if (fp != NULL) { if (fp != NULL) {
pSql->freed = 1; pSql->freed = 1;
@ -804,9 +863,7 @@ void taos_free_result_imp(TAOS_RES* res, int keepCmd) {
} }
} }
void taos_free_result(TAOS_RES *res) { void taos_free_result(TAOS_RES *res) { taos_free_result_imp(res, 0); }
taos_free_result_imp(res, 0);
}
int taos_errno(TAOS *taos) { int taos_errno(TAOS *taos) {
STscObj *pObj = (STscObj *)taos; STscObj *pObj = (STscObj *)taos;
@ -822,26 +879,24 @@ int taos_errno(TAOS *taos) {
return code; return code;
} }
static bool validErrorCode(int32_t code) { static bool validErrorCode(int32_t code) { return code >= TSDB_CODE_SUCCESS && code < TSDB_CODE_MAX_ERROR_CODE; }
return code >= TSDB_CODE_SUCCESS && code < TSDB_CODE_MAX_ERROR_CODE;
}
/* /*
* In case of invalid sql error, additional information is attached to explain * In case of invalid sql error, additional information is attached to explain
* why the sql is invalid * why the sql is invalid
*/ */
static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd* pCmd) { static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) {
if (code != TSDB_CODE_INVALID_SQL) { if (code != TSDB_CODE_INVALID_SQL) {
return false; return false;
} }
size_t len = strlen(pCmd->payload); size_t len = strlen(pCmd->payload);
char* z = NULL; char *z = NULL;
if (len > 0) { if (len > 0) {
z = strstr (pCmd->payload, "invalid SQL"); z = strstr(pCmd->payload, "invalid SQL");
} }
return z != NULL; return z != NULL;
} }
@ -851,12 +906,12 @@ char *taos_errstr(TAOS *taos) {
if (pObj == NULL || pObj->signature != pObj) return tsError[globalCode]; if (pObj == NULL || pObj->signature != pObj) return tsError[globalCode];
SSqlObj* pSql = pObj->pSql; SSqlObj *pSql = pObj->pSql;
if (validErrorCode(pSql->res.code)) { if (validErrorCode(pSql->res.code)) {
code = pSql->res.code; code = pSql->res.code;
} else { } else {
code = TSDB_CODE_OTHERS; //unknown error code = TSDB_CODE_OTHERS; // unknown error
} }
if (hasAdditionalErrorInfo(code, &pSql->cmd)) { if (hasAdditionalErrorInfo(code, &pSql->cmd)) {
@ -954,14 +1009,14 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
size_t xlen = 0; size_t xlen = 0;
for (xlen = 0; xlen <= fields[i].bytes; xlen++) { for (xlen = 0; xlen <= fields[i].bytes; xlen++) {
char c = ((char*)row[i])[xlen]; char c = ((char *)row[i])[xlen];
if (c == 0) break; if (c == 0) break;
str[len++] = c; str[len++] = c;
} }
str[len] = 0; str[len] = 0;
} break; } break;
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i]));

36
src/client/src/tscStream.c
View File

@ -246,8 +246,6 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
int32_t retry = tsProjectExecInterval; int32_t retry = tsProjectExecInterval;
tscError("%p stream:%p, retrieve no data, code:%d, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retry); tscError("%p stream:%p, retrieve no data, code:%d, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retry);
tscClearSqlMetaInfoForce(&(pStream->pSql->cmd));
tscSetRetryTimer(pStream, pStream->pSql, retry); tscSetRetryTimer(pStream, pStream->pSql, retry);
return; return;
} }
@ -381,41 +379,41 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if (pQueryInfo->nAggTimeInterval < minIntervalTime) { if (pQueryInfo->intervalTime < minIntervalTime) {
tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64 "", pSql, pStream, tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64 "", pSql, pStream,
pQueryInfo->nAggTimeInterval, minIntervalTime); pQueryInfo->intervalTime, minIntervalTime);
pQueryInfo->nAggTimeInterval = minIntervalTime; pQueryInfo->intervalTime = minIntervalTime;
} }
pStream->interval = pQueryInfo->nAggTimeInterval; // it shall be derived from sql string pStream->interval = pQueryInfo->intervalTime; // it shall be derived from sql string
if (pQueryInfo->nSlidingTime == 0) { if (pQueryInfo->slidingTime == 0) {
pQueryInfo->nSlidingTime = pQueryInfo->nAggTimeInterval; pQueryInfo->slidingTime = pQueryInfo->intervalTime;
} }
int64_t minSlidingTime = int64_t minSlidingTime =
(pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMinSlidingTime * 1000L : tsMinSlidingTime; (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMinSlidingTime * 1000L : tsMinSlidingTime;
if (pQueryInfo->nSlidingTime == -1) { if (pQueryInfo->slidingTime == -1) {
pQueryInfo->nSlidingTime = pQueryInfo->nAggTimeInterval; pQueryInfo->slidingTime = pQueryInfo->intervalTime;
} else if (pQueryInfo->nSlidingTime < minSlidingTime) { } else if (pQueryInfo->slidingTime < minSlidingTime) {
tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64 "", pSql, pStream, tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64 "", pSql, pStream,
pQueryInfo->nSlidingTime, minSlidingTime); pQueryInfo->slidingTime, minSlidingTime);
pQueryInfo->nSlidingTime = minSlidingTime; pQueryInfo->slidingTime = minSlidingTime;
} }
if (pQueryInfo->nSlidingTime > pQueryInfo->nAggTimeInterval) { if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) {
tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64 "", pSql, pStream, tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64 "", pSql, pStream,
pQueryInfo->nSlidingTime, pQueryInfo->nAggTimeInterval); pQueryInfo->slidingTime, pQueryInfo->intervalTime);
pQueryInfo->nSlidingTime = pQueryInfo->nAggTimeInterval; pQueryInfo->slidingTime = pQueryInfo->intervalTime;
} }
pStream->slidingTime = pQueryInfo->nSlidingTime; pStream->slidingTime = pQueryInfo->slidingTime;
pQueryInfo->nAggTimeInterval = 0; // clear the interval value to avoid the force time window split by query processor pQueryInfo->intervalTime = 0; // clear the interval value to avoid the force time window split by query processor
pQueryInfo->nSlidingTime = 0; pQueryInfo->slidingTime = 0;
} }
static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) { static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, int64_t stime) {

220
src/client/src/tscUtil.c
View File

@ -27,6 +27,7 @@
#include "tsclient.h" #include "tsclient.h"
#include "tsqldef.h" #include "tsqldef.h"
#include "ttimer.h" #include "ttimer.h"
#include "tast.h"
/* /*
* the detailed information regarding metric meta key is: * the detailed information regarding metric meta key is:
@ -64,7 +65,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) {
size_t redundantLen = 20; size_t redundantLen = 20;
size_t bufSize = strlen(pMeterMetaInfo->name) + tbnameCondLen + strlen(join) + strlen(tagIdBuf); size_t bufSize = strlen(pMeterMetaInfo->name) + tbnameCondLen + strlen(join) + strlen(tagIdBuf);
if (cond != NULL) { if (cond != NULL && cond->cond != NULL) {
bufSize += strlen(cond->cond); bufSize += strlen(cond->cond);
} }
@ -72,7 +73,7 @@ void tscGetMetricMetaCacheKey(SQueryInfo* pQueryInfo, char* str, uint64_t uid) {
char* tmp = calloc(1, bufSize); char* tmp = calloc(1, bufSize);
int32_t keyLen = snprintf(tmp, bufSize, "%s,%s,%s,%d,%s,[%s],%d", pMeterMetaInfo->name, int32_t keyLen = snprintf(tmp, bufSize, "%s,%s,%s,%d,%s,[%s],%d", pMeterMetaInfo->name,
(cond != NULL ? cond->cond : NULL), (tbnameCondLen > 0 ? pTagCond->tbnameCond.cond : NULL), ((cond != NULL && cond->cond != NULL) ? cond->cond : NULL), (tbnameCondLen > 0 ? pTagCond->tbnameCond.cond : NULL),
pTagCond->relType, join, tagIdBuf, pQueryInfo->groupbyExpr.orderType); pTagCond->relType, join, tagIdBuf, pQueryInfo->groupbyExpr.orderType);
assert(keyLen <= bufSize); assert(keyLen <= bufSize);
@ -202,6 +203,9 @@ SMeterSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) {
tscError("illegal sidIdx:%d, reset to 0, sidIdx range:%d-%d", idx, 0, sidRange); tscError("illegal sidIdx:%d, reset to 0, sidIdx range:%d-%d", idx, 0, sidRange);
idx = 0; idx = 0;
} }
assert(pSidList->pSidExtInfoList[idx] >= 0);
return (SMeterSidExtInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList); return (SMeterSidExtInfo*)(pSidList->pSidExtInfoList[idx] + (char*)pSidList);
} }
@ -335,35 +339,17 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) {
tfree(pQueryInfo->defaultVal); tfree(pQueryInfo->defaultVal);
} }
void tscClearSqlMetaInfoForce(SSqlCmd* pCmd) {
/* remove the metermeta/metricmeta in cache */
// taosRemoveDataFromCache(tscCacheHandle, (void**)&(pCmd->pMeterMeta), true);
// taosRemoveDataFromCache(tscCacheHandle, (void**)&(pCmd->pMetricMeta), true);
}
int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
if (pRes->tsrow == NULL) { if (pRes->tsrow == NULL) {
pRes->numOfnchar = 0;
int32_t numOfOutputCols = pQueryInfo->fieldsInfo.numOfOutputCols; int32_t numOfOutputCols = pQueryInfo->fieldsInfo.numOfOutputCols;
for (int32_t i = 0; i < numOfOutputCols; ++i) { pRes->numOfCols = numOfOutputCols;
TAOS_FIELD* pField = tscFieldInfoGetField(pQueryInfo, i);
if (pField->type == TSDB_DATA_TYPE_NCHAR) {
pRes->numOfnchar++;
}
}
pRes->tsrow = calloc(1, (POINTER_BYTES + sizeof(short)) * numOfOutputCols + POINTER_BYTES * pRes->numOfnchar); pRes->tsrow = calloc(POINTER_BYTES, numOfOutputCols);
pRes->bytes = calloc(numOfOutputCols, sizeof(short)); pRes->buffer = calloc(POINTER_BYTES, numOfOutputCols);
if (pRes->numOfnchar > 0) {
pRes->buffer = calloc(POINTER_BYTES, pRes->numOfnchar);
}
// not enough memory // not enough memory
if (pRes->tsrow == NULL || pRes->bytes == NULL || (pRes->buffer == NULL && pRes->numOfnchar > 0)) { if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) {
tfree(pRes->tsrow); tfree(pRes->tsrow);
tfree(pRes->bytes);
tfree(pRes->buffer); tfree(pRes->buffer);
pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
@ -376,13 +362,12 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
void tscDestroyResPointerInfo(SSqlRes* pRes) { void tscDestroyResPointerInfo(SSqlRes* pRes) {
if (pRes->buffer != NULL) { if (pRes->buffer != NULL) {
assert(pRes->numOfnchar > 0);
// free all buffers containing the multibyte string // free all buffers containing the multibyte string
for (int i = 0; i < pRes->numOfnchar; i++) { for (int i = 0; i < pRes->numOfCols; i++) {
tfree(pRes->buffer[i]); tfree(pRes->buffer[i]);
} }
pRes->numOfnchar = 0; pRes->numOfCols = 0;
} }
tfree(pRes->pRsp); tfree(pRes->pRsp);
@ -391,7 +376,6 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) {
tfree(pRes->pGroupRec); tfree(pRes->pGroupRec);
tfree(pRes->pColumnIndex); tfree(pRes->pColumnIndex);
tfree(pRes->buffer); tfree(pRes->buffer);
tfree(pRes->bytes);
pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free
} }
@ -831,11 +815,18 @@ static void ensureSpace(SFieldInfo* pFieldInfo, int32_t size) {
pFieldInfo->pFields = realloc(pFieldInfo->pFields, newSize * sizeof(TAOS_FIELD)); pFieldInfo->pFields = realloc(pFieldInfo->pFields, newSize * sizeof(TAOS_FIELD));
memset(&pFieldInfo->pFields[oldSize], 0, inc * sizeof(TAOS_FIELD)); memset(&pFieldInfo->pFields[oldSize], 0, inc * sizeof(TAOS_FIELD));
pFieldInfo->pOffset = realloc(pFieldInfo->pOffset, newSize * sizeof(int16_t)); // pFieldInfo->pOffset = realloc(pFieldInfo->pOffset, newSize * sizeof(int16_t));
memset(&pFieldInfo->pOffset[oldSize], 0, inc * sizeof(int16_t)); // memset(&pFieldInfo->pOffset[oldSize], 0, inc * sizeof(int16_t));
pFieldInfo->pVisibleCols = realloc(pFieldInfo->pVisibleCols, newSize * sizeof(bool)); pFieldInfo->pVisibleCols = realloc(pFieldInfo->pVisibleCols, newSize * sizeof(bool));
memset(&pFieldInfo->pVisibleCols[oldSize], 0, inc * sizeof(bool));
pFieldInfo->pSqlExpr = realloc(pFieldInfo->pSqlExpr, POINTER_BYTES*newSize);
pFieldInfo->pExpr = realloc(pFieldInfo->pExpr, POINTER_BYTES*newSize);
memset(&pFieldInfo->pSqlExpr[oldSize], 0, inc * POINTER_BYTES);
memset(&pFieldInfo->pExpr[oldSize], 0, inc * POINTER_BYTES);
pFieldInfo->numOfAlloc = newSize; pFieldInfo->numOfAlloc = newSize;
} }
} }
@ -844,6 +835,15 @@ static void evic(SFieldInfo* pFieldInfo, int32_t index) {
if (index < pFieldInfo->numOfOutputCols) { if (index < pFieldInfo->numOfOutputCols) {
memmove(&pFieldInfo->pFields[index + 1], &pFieldInfo->pFields[index], memmove(&pFieldInfo->pFields[index + 1], &pFieldInfo->pFields[index],
sizeof(pFieldInfo->pFields[0]) * (pFieldInfo->numOfOutputCols - index)); sizeof(pFieldInfo->pFields[0]) * (pFieldInfo->numOfOutputCols - index));
memmove(&pFieldInfo->pVisibleCols[index + 1], &pFieldInfo->pVisibleCols[index],
sizeof(pFieldInfo->pVisibleCols[0]) * (pFieldInfo->numOfOutputCols - index));
memmove(&pFieldInfo->pSqlExpr[index + 1], &pFieldInfo->pSqlExpr[index],
sizeof(pFieldInfo->pSqlExpr[0]) * (pFieldInfo->numOfOutputCols - index));
memmove(&pFieldInfo->pExpr[index + 1], &pFieldInfo->pExpr[index],
sizeof(pFieldInfo->pExpr[0]) * (pFieldInfo->numOfOutputCols - index));
} }
} }
@ -868,7 +868,6 @@ void tscFieldInfoSetValFromField(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIE
memcpy(&pFieldInfo->pFields[index], pField, sizeof(TAOS_FIELD)); memcpy(&pFieldInfo->pFields[index], pField, sizeof(TAOS_FIELD));
pFieldInfo->pVisibleCols[index] = true; pFieldInfo->pVisibleCols[index] = true;
pFieldInfo->numOfOutputCols++; pFieldInfo->numOfOutputCols++;
} }
@ -902,29 +901,49 @@ void tscFieldInfoSetValue(SFieldInfo* pFieldInfo, int32_t index, int8_t type, co
pFieldInfo->numOfOutputCols++; pFieldInfo->numOfOutputCols++;
} }
void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo) { void tscFieldInfoSetExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlExpr* pExpr) {
SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; assert(index >= 0 && index < pFieldInfo->numOfOutputCols);
pFieldInfo->pOffset[0] = 0; pFieldInfo->pSqlExpr[index] = pExpr;
}
for (int32_t i = 1; i < pFieldInfo->numOfOutputCols; ++i) { void tscFieldInfoSetBinExpr(SFieldInfo* pFieldInfo, int32_t index, SSqlFunctionExpr* pExpr) {
pFieldInfo->pOffset[i] = pFieldInfo->pOffset[i - 1] + pFieldInfo->pFields[i - 1].bytes; assert(index >= 0 && index < pFieldInfo->numOfOutputCols);
pFieldInfo->pExpr[index] = pExpr;
}
void tscFieldInfoCalOffset(SQueryInfo* pQueryInfo) {
SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo;
pExprInfo->pExprs[0]->offset = 0;
for (int32_t i = 1; i < pExprInfo->numOfExprs; ++i) {
pExprInfo->pExprs[i]->offset = pExprInfo->pExprs[i - 1]->offset + pExprInfo->pExprs[i - 1]->resBytes;
} }
} }
void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) { void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) {
SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; // SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo;
if (pFieldInfo->numOfOutputCols == 0) { // if (pFieldInfo->numOfOutputCols == 0) {
// return;
// }
//
// pFieldInfo->pOffset[0] = 0;
//
// /*
// * the retTypeLen is used to store the intermediate result length
// * for potential secondary merge exists
// */
// for (int32_t i = 1; i < pFieldInfo->numOfOutputCols; ++i) {
// pFieldInfo->pOffset[i] = pFieldInfo->pOffset[i - 1] + tscSqlExprGet(pQueryInfo, i - 1)->resBytes;
// }
SSqlExprInfo* pExprInfo = &pQueryInfo->exprsInfo;
if (pExprInfo->numOfExprs == 0) {
return; return;
} }
pFieldInfo->pOffset[0] = 0; pExprInfo->pExprs[0]->offset = 0;
/* for (int32_t i = 1; i < pExprInfo->numOfExprs; ++i) {
* the retTypeLen is used to store the intermediate result length pExprInfo->pExprs[i]->offset = pExprInfo->pExprs[i - 1]->offset + pExprInfo->pExprs[i - 1]->resBytes;
* for potential secondary merge exists
*/
for (int32_t i = 1; i < pFieldInfo->numOfOutputCols; ++i) {
pFieldInfo->pOffset[i] = pFieldInfo->pOffset[i - 1] + tscSqlExprGet(pQueryInfo, i - 1)->resBytes;
} }
} }
@ -940,6 +959,8 @@ void tscFieldInfoCopy(SFieldInfo* src, SFieldInfo* dst, const int32_t* indexList
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
assert(indexList[i] >= 0 && indexList[i] <= src->numOfOutputCols); assert(indexList[i] >= 0 && indexList[i] <= src->numOfOutputCols);
tscFieldInfoSetValFromField(dst, i, &src->pFields[indexList[i]]); tscFieldInfoSetValFromField(dst, i, &src->pFields[indexList[i]]);
dst->pVisibleCols[i] = src->pVisibleCols[indexList[i]];
dst->pSqlExpr[i] = src->pSqlExpr[indexList[i]];
} }
} }
} }
@ -948,12 +969,14 @@ void tscFieldInfoCopyAll(SFieldInfo* dst, SFieldInfo* src) {
*dst = *src; *dst = *src;
dst->pFields = malloc(sizeof(TAOS_FIELD) * dst->numOfAlloc); dst->pFields = malloc(sizeof(TAOS_FIELD) * dst->numOfAlloc);
dst->pOffset = malloc(sizeof(short) * dst->numOfAlloc);
dst->pVisibleCols = malloc(sizeof(bool) * dst->numOfAlloc); dst->pVisibleCols = malloc(sizeof(bool) * dst->numOfAlloc);
dst->pSqlExpr = malloc(POINTER_BYTES * dst->numOfAlloc);
dst->pExpr = malloc(POINTER_BYTES * dst->numOfAlloc);
memcpy(dst->pFields, src->pFields, sizeof(TAOS_FIELD) * dst->numOfOutputCols); memcpy(dst->pFields, src->pFields, sizeof(TAOS_FIELD) * dst->numOfOutputCols);
memcpy(dst->pOffset, src->pOffset, sizeof(short) * dst->numOfOutputCols);
memcpy(dst->pVisibleCols, src->pVisibleCols, sizeof(bool) * dst->numOfOutputCols); memcpy(dst->pVisibleCols, src->pVisibleCols, sizeof(bool) * dst->numOfOutputCols);
memcpy(dst->pSqlExpr, src->pSqlExpr, POINTER_BYTES * dst->numOfOutputCols);
memcpy(dst->pExpr, src->pExpr, POINTER_BYTES * dst->numOfOutputCols);
} }
TAOS_FIELD* tscFieldInfoGetField(SQueryInfo* pQueryInfo, int32_t index) { TAOS_FIELD* tscFieldInfoGetField(SQueryInfo* pQueryInfo, int32_t index) {
@ -967,11 +990,11 @@ TAOS_FIELD* tscFieldInfoGetField(SQueryInfo* pQueryInfo, int32_t index) {
int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutputCols; } int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutputCols; }
int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) {
if (index >= pQueryInfo->fieldsInfo.numOfOutputCols) { if (index >= pQueryInfo->exprsInfo.numOfExprs) {
return 0; return 0;
} }
return pQueryInfo->fieldsInfo.pOffset[index]; return pQueryInfo->exprsInfo.pExprs[index]->offset;
} }
int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2) { int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2) {
@ -995,13 +1018,16 @@ int32_t tscFieldInfoCompare(SFieldInfo* pFieldInfo1, SFieldInfo* pFieldInfo2) {
} }
int32_t tscGetResRowLength(SQueryInfo* pQueryInfo) { int32_t tscGetResRowLength(SQueryInfo* pQueryInfo) {
SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; if (pQueryInfo->exprsInfo.numOfExprs <= 0) {
if (pFieldInfo->numOfOutputCols <= 0) {
return 0; return 0;
} }
return pFieldInfo->pOffset[pFieldInfo->numOfOutputCols - 1] + int32_t size = 0;
pFieldInfo->pFields[pFieldInfo->numOfOutputCols - 1].bytes; for(int32_t i = 0; i < pQueryInfo->exprsInfo.numOfExprs; ++i) {
size += pQueryInfo->exprsInfo.pExprs[i]->resBytes;
}
return size;
} }
void tscClearFieldInfo(SFieldInfo* pFieldInfo) { void tscClearFieldInfo(SFieldInfo* pFieldInfo) {
@ -1009,10 +1035,19 @@ void tscClearFieldInfo(SFieldInfo* pFieldInfo) {
return; return;
} }
tfree(pFieldInfo->pOffset);
tfree(pFieldInfo->pFields); tfree(pFieldInfo->pFields);
tfree(pFieldInfo->pVisibleCols); tfree(pFieldInfo->pVisibleCols);
tfree(pFieldInfo->pSqlExpr);
for(int32_t i = 0; i < pFieldInfo->numOfOutputCols; ++i) {
if (pFieldInfo->pExpr[i] != NULL) {
tSQLBinaryExprDestroy(&pFieldInfo->pExpr[i]->pBinExprInfo.pBinExpr, NULL);
tfree(pFieldInfo->pExpr[i]->pBinExprInfo.pReqColumns);
tfree(pFieldInfo->pExpr[i]);
}
}
tfree(pFieldInfo->pExpr);
memset(pFieldInfo, 0, sizeof(SFieldInfo)); memset(pFieldInfo, 0, sizeof(SFieldInfo));
} }
@ -1050,11 +1085,12 @@ SSqlExpr* tscSqlExprInsertEmpty(SQueryInfo* pQueryInfo, int32_t index, int16_t f
_exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1); _exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1);
_exprEvic(pExprInfo, index); _exprEvic(pExprInfo, index);
SSqlExpr* pExpr = &pExprInfo->pExprs[index]; SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr));
pExpr->functionId = functionId; pExpr->functionId = functionId;
pExprInfo->numOfExprs++; pExprInfo->numOfExprs++;
pExprInfo->pExprs[index] = pExpr;
return pExpr; return pExpr;
} }
@ -1067,8 +1103,9 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi
_exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1); _exprCheckSpace(pExprInfo, pExprInfo->numOfExprs + 1);
_exprEvic(pExprInfo, index); _exprEvic(pExprInfo, index);
SSqlExpr* pExpr = &pExprInfo->pExprs[index]; SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr));
pExprInfo->pExprs[index] = pExpr;
pExpr->functionId = functionId; pExpr->functionId = functionId;
int16_t numOfCols = pMeterMetaInfo->pMeterMeta->numOfColumns; int16_t numOfCols = pMeterMetaInfo->pMeterMeta->numOfColumns;
@ -1110,7 +1147,7 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi
return NULL; return NULL;
} }
SSqlExpr* pExpr = &pExprInfo->pExprs[index]; SSqlExpr* pExpr = pExprInfo->pExprs[index];
pExpr->functionId = functionId; pExpr->functionId = functionId;
@ -1123,6 +1160,10 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi
return pExpr; return pExpr;
} }
int32_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) {
return pQueryInfo->exprsInfo.numOfExprs;
}
void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex) { void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes, int16_t tableIndex) {
if (pExpr == NULL || argument == NULL || bytes == 0) { if (pExpr == NULL || argument == NULL || bytes == 0) {
return; return;
@ -1141,7 +1182,7 @@ SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index) {
return NULL; return NULL;
} }
return &pQueryInfo->exprsInfo.pExprs[index]; return pQueryInfo->exprsInfo.pExprs[index];
} }
void* tscSqlExprDestroy(SSqlExpr* pExpr) { void* tscSqlExprDestroy(SSqlExpr* pExpr) {
@ -1153,6 +1194,8 @@ void* tscSqlExprDestroy(SSqlExpr* pExpr) {
tVariantDestroy(&pExpr->param[i]); tVariantDestroy(&pExpr->param[i]);
} }
tfree(pExpr);
return NULL; return NULL;
} }
@ -1164,8 +1207,8 @@ void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo) {
return; return;
} }
for(int32_t i = 0; i < pExprInfo->numOfAlloc; ++i) { for(int32_t i = 0; i < pExprInfo->numOfExprs; ++i) {
tscSqlExprDestroy(&pExprInfo->pExprs[i]); tscSqlExprDestroy(pExprInfo->pExprs[i]);
} }
tfree(pExprInfo->pExprs); tfree(pExprInfo->pExprs);
@ -1175,27 +1218,40 @@ void tscSqlExprInfoDestroy(SSqlExprInfo* pExprInfo) {
} }
void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t tableuid) { void tscSqlExprCopy(SSqlExprInfo* dst, const SSqlExprInfo* src, uint64_t tableuid, bool deepcopy) {
if (src == NULL) { if (src == NULL) {
return; return;
} }
*dst = *src; *dst = *src;
dst->pExprs = calloc(dst->numOfAlloc, sizeof(SSqlExpr)); dst->pExprs = calloc(dst->numOfAlloc, POINTER_BYTES);
int16_t num = 0; int16_t num = 0;
for (int32_t i = 0; i < src->numOfExprs; ++i) { for (int32_t i = 0; i < src->numOfExprs; ++i) {
if (src->pExprs[i].uid == tableuid) { if (src->pExprs[i]->uid == tableuid) {
dst->pExprs[num++] = src->pExprs[i];
if (deepcopy) {
dst->pExprs[num] = calloc(1, sizeof(SSqlExpr));
*dst->pExprs[num] = *src->pExprs[i];
} else {
dst->pExprs[num] = src->pExprs[i];
}
num++;
} }
} }
dst->numOfExprs = num; dst->numOfExprs = num;
for (int32_t i = 0; i < dst->numOfExprs; ++i) {
for (int32_t j = 0; j < src->pExprs[i].numOfParams; ++j) { if (deepcopy) {
tVariantAssign(&dst->pExprs[i].param[j], &src->pExprs[i].param[j]); for (int32_t i = 0; i < dst->numOfExprs; ++i) {
for (int32_t j = 0; j < src->pExprs[i]->numOfParams; ++j) {
tVariantAssign(&dst->pExprs[i]->param[j], &src->pExprs[i]->param[j]);
}
} }
} }
} }
static void clearVal(SColumnBase* pBase) { static void clearVal(SColumnBase* pBase) {
@ -1950,7 +2006,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
} }
uint64_t uid = pMeterMetaInfo->pMeterMeta->uid; uint64_t uid = pMeterMetaInfo->pMeterMeta->uid;
tscSqlExprCopy(&pNewQueryInfo->exprsInfo, &pQueryInfo->exprsInfo, uid); tscSqlExprCopy(&pNewQueryInfo->exprsInfo, &pQueryInfo->exprsInfo, uid, true);
int32_t numOfOutputCols = pNewQueryInfo->exprsInfo.numOfExprs; int32_t numOfOutputCols = pNewQueryInfo->exprsInfo.numOfExprs;
@ -1965,7 +2021,19 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pNewQueryInfo->fieldsInfo, indexList, numOfOutputCols); tscFieldInfoCopy(&pQueryInfo->fieldsInfo, &pNewQueryInfo->fieldsInfo, indexList, numOfOutputCols);
free(indexList); free(indexList);
// make sure the the sqlExpr for each fields is correct
// todo handle the agg arithmetic expression
for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutputCols; ++f) {
char* name = pNewQueryInfo->fieldsInfo.pFields[f].name;
for(int32_t k1 = 0; k1 < pNewQueryInfo->exprsInfo.numOfExprs; ++k1) {
SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1);
if (strcmp(name, pExpr1->aliasName) == 0) {
pNewQueryInfo->fieldsInfo.pSqlExpr[f] = pExpr1;
}
}
}
tscFieldInfoUpdateOffsetForInterResult(pNewQueryInfo); tscFieldInfoUpdateOffsetForInterResult(pNewQueryInfo);
} }

9
src/inc/hash.h
View File

@ -16,6 +16,10 @@
#ifndef TDENGINE_HASH_H #ifndef TDENGINE_HASH_H
#define TDENGINE_HASH_H #define TDENGINE_HASH_H
#ifdef __cplusplus
extern "C" {
#endif
#include "hashutil.h" #include "hashutil.h"
#define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define HASH_MAX_CAPACITY (1024 * 1024 * 16)
@ -64,11 +68,12 @@ int32_t taosNumElemsInHashTable(HashObj *pObj);
char *taosGetDataFromHashTable(HashObj *pObj, const char *key, uint32_t keyLen); char *taosGetDataFromHashTable(HashObj *pObj, const char *key, uint32_t keyLen);
void taosCleanUpHashTable(void *handle); void taosCleanUpHashTable(void *handle);
int32_t taosGetHashMaxOverflowLength(HashObj *pObj); int32_t taosGetHashMaxOverflowLength(HashObj *pObj);
int32_t taosCheckHashTable(HashObj *pObj); #ifdef __cplusplus
}
#endif
#endif // TDENGINE_HASH_H #endif // TDENGINE_HASH_H

3
src/inc/taosmsg.h
View File

@ -417,6 +417,7 @@ typedef struct SColIndexEx {
int16_t colIdx; int16_t colIdx;
int16_t colIdxInBuf; int16_t colIdxInBuf;
uint16_t flag; // denote if it is a tag or not uint16_t flag; // denote if it is a tag or not
char name[TSDB_COL_NAME_LEN];
} SColIndexEx; } SColIndexEx;
/* sql function msg, to describe the message to vnode about sql function /* sql function msg, to describe the message to vnode about sql function
@ -514,7 +515,7 @@ typedef struct {
int16_t numOfCols; // the number of columns will be load from vnode int16_t numOfCols; // the number of columns will be load from vnode
char intervalTimeUnit; // time interval type, for revisement of interval(1d) char intervalTimeUnit; // time interval type, for revisement of interval(1d)
int64_t nAggTimeInterval; // time interval for aggregation, in million second int64_t intervalTime; // time interval for aggregation, in million second
int64_t slidingTime; // value for sliding window int64_t slidingTime; // value for sliding window
// tag schema, used to parse tag information in pSidExtInfo // tag schema, used to parse tag information in pSidExtInfo

2
src/inc/tast.h
View File

@ -103,6 +103,8 @@ void tSQLBinaryExprCalcTraverse(tSQLBinaryExpr *pExprs, int32_t numOfRows, char
void tSQLBinaryExprTrv(tSQLBinaryExpr *pExprs, int32_t *val, int16_t *ids); void tSQLBinaryExprTrv(tSQLBinaryExpr *pExprs, int32_t *val, int16_t *ids);
void tQueryResultClean(tQueryResultset *pRes); void tQueryResultClean(tQueryResultset *pRes);
uint8_t getBinaryExprOptr(SSQLToken *pToken);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

20
src/inc/tresultBuf.h
View File

@ -14,7 +14,7 @@ typedef struct SIDList {
int32_t* pData; int32_t* pData;
} SIDList; } SIDList;
typedef struct SQueryResultBuf { typedef struct SQueryDiskbasedResultBuf {
int32_t numOfRowsPerPage; int32_t numOfRowsPerPage;
int32_t numOfPages; int32_t numOfPages;
int64_t totalBufSize; int64_t totalBufSize;
@ -27,7 +27,7 @@ typedef struct SQueryResultBuf {
uint32_t numOfAllocGroupIds; // number of allocated id list uint32_t numOfAllocGroupIds; // number of allocated id list
void* idsTable; // id hash table void* idsTable; // id hash table
SIDList* list; // for each id, there is a page id list SIDList* list; // for each id, there is a page id list
} SQueryResultBuf; } SQueryDiskbasedResultBuf;
/** /**
* create disk-based result buffer * create disk-based result buffer
@ -36,7 +36,7 @@ typedef struct SQueryResultBuf {
* @param rowSize * @param rowSize
* @return * @return
*/ */
int32_t createResultBuf(SQueryResultBuf** pResultBuf, int32_t size, int32_t rowSize); int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize);
/** /**
* *
@ -45,14 +45,14 @@ int32_t createResultBuf(SQueryResultBuf** pResultBuf, int32_t size, int32_t rowS
* @param pageId * @param pageId
* @return * @return
*/ */
tFilePage* getNewDataBuf(SQueryResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); tFilePage* getNewDataBuf(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId);
/** /**
* *
* @param pResultBuf * @param pResultBuf
* @return * @return
*/ */
int32_t getNumOfRowsPerPage(SQueryResultBuf* pResultBuf); int32_t getNumOfRowsPerPage(SQueryDiskbasedResultBuf* pResultBuf);
/** /**
* *
@ -60,7 +60,7 @@ int32_t getNumOfRowsPerPage(SQueryResultBuf* pResultBuf);
* @param groupId * @param groupId
* @return * @return
*/ */
SIDList getDataBufPagesIdList(SQueryResultBuf* pResultBuf, int32_t groupId); SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId);
/** /**
* get the specified buffer page by id * get the specified buffer page by id
@ -68,27 +68,27 @@ SIDList getDataBufPagesIdList(SQueryResultBuf* pResultBuf, int32_t groupId);
* @param id * @param id
* @return * @return
*/ */
tFilePage* getResultBufferPageById(SQueryResultBuf* pResultBuf, int32_t id); tFilePage* getResultBufferPageById(SQueryDiskbasedResultBuf* pResultBuf, int32_t id);
/** /**
* get the total buffer size in the format of disk file * get the total buffer size in the format of disk file
* @param pResultBuf * @param pResultBuf
* @return * @return
*/ */
int32_t getResBufSize(SQueryResultBuf* pResultBuf); int32_t getResBufSize(SQueryDiskbasedResultBuf* pResultBuf);
/** /**
* get the number of groups in the result buffer * get the number of groups in the result buffer
* @param pResultBuf * @param pResultBuf
* @return * @return
*/ */
int32_t getNumOfResultBufGroupId(SQueryResultBuf* pResultBuf); int32_t getNumOfResultBufGroupId(SQueryDiskbasedResultBuf* pResultBuf);
/** /**
* destroy result buffer * destroy result buffer
* @param pResultBuf * @param pResultBuf
*/ */
void destroyResultBuf(SQueryResultBuf* pResultBuf); void destroyResultBuf(SQueryDiskbasedResultBuf* pResultBuf);
/** /**
* *

14
src/inc/tsqlfunction.h
View File

@ -107,10 +107,11 @@ extern "C" {
#define TOP_BOTTOM_QUERY_LIMIT 100 #define TOP_BOTTOM_QUERY_LIMIT 100
enum { enum {
MASTER_SCAN = 0x0, MASTER_SCAN = 0x0u,
SUPPLEMENTARY_SCAN = 0x1, SUPPLEMENTARY_SCAN = 0x1u,
FIRST_STAGE_MERGE = 0x10, REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan
SECONDARY_STAGE_MERGE = 0x20, FIRST_STAGE_MERGE = 0x10u,
SECONDARY_STAGE_MERGE = 0x20u,
}; };
#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) #define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0)
@ -171,7 +172,7 @@ typedef struct SQLFunctionCtx {
int32_t startOffset; int32_t startOffset;
int32_t size; // number of rows int32_t size; // number of rows
int32_t order; // asc|desc int32_t order; // asc|desc
int32_t scanFlag; // TODO merge with currentStage uint32_t scanFlag; // TODO merge with currentStage
int16_t inputType; int16_t inputType;
int16_t inputBytes; int16_t inputBytes;
@ -304,6 +305,9 @@ void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, in
bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval); bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval);
bool stableQueryFunctChanged(int32_t funcId);
void resetResultInfo(SResultInfo *pResInfo); void resetResultInfo(SResultInfo *pResInfo);
void initResultInfo(SResultInfo *pResInfo); void initResultInfo(SResultInfo *pResInfo);
void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable); void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable);

2
src/system/detail/inc/vnode.h
View File

@ -259,7 +259,7 @@ typedef struct SQuery {
int64_t blockId; int64_t blockId;
TSKEY skey; TSKEY skey;
TSKEY ekey; TSKEY ekey;
int64_t nAggTimeInterval; int64_t intervalTime;
int64_t slidingTime; // sliding time for sliding window query int64_t slidingTime; // sliding time for sliding window query
char intervalTimeUnit; // interval data type, used for daytime revise char intervalTimeUnit; // interval data type, used for daytime revise
int8_t precision; int8_t precision;

116
src/system/detail/inc/vnodeQueryImpl.h
View File

@ -25,7 +25,7 @@ extern "C" {
#include "hash.h" #include "hash.h"
#include "hashutil.h" #include "hashutil.h"
#define GET_QINFO_ADDR(x) ((char*)(x)-offsetof(SQInfo, query)) #define GET_QINFO_ADDR(x) ((char*)(x)-offsetof(SQInfo, query))
#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0)
/* /*
@ -33,10 +33,10 @@ extern "C" {
* The page size should be sufficient for at least one output result or intermediate result. * The page size should be sufficient for at least one output result or intermediate result.
* Some intermediate results may be extremely large, such as top/bottom(100) query. * Some intermediate results may be extremely large, such as top/bottom(100) query.
*/ */
#define DEFAULT_INTERN_BUF_SIZE 16384L #define DEFAULT_INTERN_BUF_SIZE 16384L
#define INIT_ALLOCATE_DISK_PAGES 60L #define INIT_ALLOCATE_DISK_PAGES 60L
#define DEFAULT_DATA_FILE_MAPPING_PAGES 2L #define DEFAULT_DATA_FILE_MAPPING_PAGES 2L
#define DEFAULT_DATA_FILE_MMAP_WINDOW_SIZE (DEFAULT_DATA_FILE_MAPPING_PAGES * DEFAULT_INTERN_BUF_SIZE) #define DEFAULT_DATA_FILE_MMAP_WINDOW_SIZE (DEFAULT_DATA_FILE_MAPPING_PAGES * DEFAULT_INTERN_BUF_SIZE)
#define IO_ENGINE_MMAP 0 #define IO_ENGINE_MMAP 0
@ -56,7 +56,7 @@ typedef enum {
* the program will call this function again, if this status is set. * the program will call this function again, if this status is set.
* used to transfer from QUERY_RESBUF_FULL * used to transfer from QUERY_RESBUF_FULL
*/ */
QUERY_NOT_COMPLETED = 0x1, QUERY_NOT_COMPLETED = 0x1u,
/* /*
* output buffer is full, so, the next query will be employed, * output buffer is full, so, the next query will be employed,
@ -66,7 +66,7 @@ typedef enum {
* this status is only exist in group-by clause and * this status is only exist in group-by clause and
* diff/add/division/multiply/ query. * diff/add/division/multiply/ query.
*/ */
QUERY_RESBUF_FULL = 0x2, QUERY_RESBUF_FULL = 0x2u,
/* /*
* query is over * query is over
@ -76,14 +76,13 @@ typedef enum {
* 2. when the query range on timestamp is satisfied, it is also denoted as * 2. when the query range on timestamp is satisfied, it is also denoted as
* query_compeleted * query_compeleted
*/ */
QUERY_COMPLETED = 0x4, QUERY_COMPLETED = 0x4u,
/* /*
* all data has been scanned, so current search is stopped, * all data has been scanned, so current search is stopped,
* At last, the function will transfer this status to QUERY_COMPLETED * At last, the function will transfer this status to QUERY_COMPLETED
*/ */
QUERY_NO_DATA_TO_CHECK = 0x8, QUERY_NO_DATA_TO_CHECK = 0x8u,
} vnodeQueryStatus; } vnodeQueryStatus;
typedef struct SPointInterpoSupporter { typedef struct SPointInterpoSupporter {
@ -112,15 +111,15 @@ typedef enum {
DISK_DATA_DISCARDED = 0x01, DISK_DATA_DISCARDED = 0x01,
} vnodeDiskLoadStatus; } vnodeDiskLoadStatus;
#define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) #define IS_MASTER_SCAN(runtime) (((runtime)->scanFlag & 1u) == MASTER_SCAN)
#define IS_SUPPLEMENT_SCAN(runtime) (!IS_MASTER_SCAN(runtime)) #define IS_SUPPLEMENT_SCAN(runtime) ((runtime)->scanFlag == SUPPLEMENTARY_SCAN)
#define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN) #define SET_SUPPLEMENT_SCAN_FLAG(runtime) ((runtime)->scanFlag = SUPPLEMENTARY_SCAN)
#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) #define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN)
typedef int (*__block_search_fn_t)(char* data, int num, int64_t key, int order); typedef int (*__block_search_fn_t)(char* data, int num, int64_t key, int order);
static FORCE_INLINE SMeterObj* getMeterObj(void* hashHandle, int32_t sid) { static FORCE_INLINE SMeterObj* getMeterObj(void* hashHandle, int32_t sid) {
return *(SMeterObj**)taosGetDataFromHashTable(hashHandle, (const char*) &sid, sizeof(sid)); return *(SMeterObj**)taosGetDataFromHashTable(hashHandle, (const char*)&sid, sizeof(sid));
} }
bool isQueryKilled(SQuery* pQuery); bool isQueryKilled(SQuery* pQuery);
@ -130,7 +129,7 @@ bool isSumAvgRateQuery(SQuery *pQuery);
bool isTopBottomQuery(SQuery* pQuery); bool isTopBottomQuery(SQuery* pQuery);
bool isFirstLastRowQuery(SQuery* pQuery); bool isFirstLastRowQuery(SQuery* pQuery);
bool isTSCompQuery(SQuery* pQuery); bool isTSCompQuery(SQuery* pQuery);
bool notHasQueryTimeRange(SQuery *pQuery); bool notHasQueryTimeRange(SQuery* pQuery);
bool needSupplementaryScan(SQuery* pQuery); bool needSupplementaryScan(SQuery* pQuery);
bool onDemandLoadDatablock(SQuery* pQuery, int16_t queryRangeSet); bool onDemandLoadDatablock(SQuery* pQuery, int16_t queryRangeSet);
@ -149,16 +148,15 @@ void vnodeScanAllData(SQueryRuntimeEnv* pRuntimeEnv);
int32_t vnodeQueryResultInterpolate(SQInfo* pQInfo, tFilePage** pDst, tFilePage** pDataSrc, int32_t numOfRows, int32_t vnodeQueryResultInterpolate(SQInfo* pQInfo, tFilePage** pDst, tFilePage** pDataSrc, int32_t numOfRows,
int32_t* numOfInterpo); int32_t* numOfInterpo);
void copyResToQueryResultBuf(SMeterQuerySupportObj* pSupporter, SQuery* pQuery); void copyResToQueryResultBuf(STableQuerySupportObj* pSupporter, SQuery* pQuery);
void doSkipResults(SQueryRuntimeEnv* pRuntimeEnv); void doSkipResults(SQueryRuntimeEnv* pRuntimeEnv);
void doFinalizeResult(SQueryRuntimeEnv* pRuntimeEnv); void doFinalizeResult(SQueryRuntimeEnv* pRuntimeEnv);
int64_t getNumOfResult(SQueryRuntimeEnv* pRuntimeEnv); int64_t getNumOfResult(SQueryRuntimeEnv* pRuntimeEnv);
void forwardIntervalQueryRange(SMeterQuerySupportObj* pSupporter, SQueryRuntimeEnv* pRuntimeEnv);
void forwardQueryStartPosition(SQueryRuntimeEnv* pRuntimeEnv); void forwardQueryStartPosition(SQueryRuntimeEnv* pRuntimeEnv);
bool normalizedFirstQueryRange(bool dataInDisk, bool dataInCache, SMeterQuerySupportObj* pSupporter, bool normalizedFirstQueryRange(bool dataInDisk, bool dataInCache, STableQuerySupportObj* pSupporter,
SPointInterpoSupporter* pPointInterpSupporter, int64_t* key); SPointInterpoSupporter* pPointInterpSupporter, int64_t* key);
void pointInterpSupporterInit(SQuery* pQuery, SPointInterpoSupporter* pInterpoSupport); void pointInterpSupporterInit(SQuery* pQuery, SPointInterpoSupporter* pInterpoSupport);
@ -166,41 +164,42 @@ void pointInterpSupporterDestroy(SPointInterpoSupporter* pPointInterpSupport);
void pointInterpSupporterSetData(SQInfo* pQInfo, SPointInterpoSupporter* pPointInterpSupport); void pointInterpSupporterSetData(SQInfo* pQInfo, SPointInterpoSupporter* pPointInterpSupport);
int64_t loadRequiredBlockIntoMem(SQueryRuntimeEnv* pRuntimeEnv, SPositionInfo* position); int64_t loadRequiredBlockIntoMem(SQueryRuntimeEnv* pRuntimeEnv, SPositionInfo* position);
int32_t doCloseAllOpenedResults(SMeterQuerySupportObj* pSupporter); void disableFunctForSuppleScan(STableQuerySupportObj* pSupporter, int32_t order);
void disableFunctForSuppleScan(SQueryRuntimeEnv* pRuntimeEnv, int32_t order); void enableFunctForMasterScan(SQueryRuntimeEnv* pRuntimeEnv, int32_t order);
void enableFunctForMasterScan(SQueryRuntimeEnv* pRuntimeEnv, int32_t order);
int32_t mergeMetersResultToOneGroups(SMeterQuerySupportObj* pSupporter); int32_t mergeMetersResultToOneGroups(STableQuerySupportObj* pSupporter);
void copyFromGroupBuf(SQInfo* pQInfo, SOutputRes* result); void copyFromWindowResToSData(SQInfo* pQInfo, SWindowResult* result);
SBlockInfo getBlockInfo(SQueryRuntimeEnv *pRuntimeEnv);
SBlockInfo getBlockBasicInfo(SQueryRuntimeEnv *pRuntimeEnv, void* pBlock, int32_t type);
SBlockInfo getBlockBasicInfo(SQueryRuntimeEnv* pRuntimeEnv, void* pBlock, int32_t blockType);
SCacheBlock* getCacheDataBlock(SMeterObj* pMeterObj, SQueryRuntimeEnv* pRuntimeEnv, int32_t slot); SCacheBlock* getCacheDataBlock(SMeterObj* pMeterObj, SQueryRuntimeEnv* pRuntimeEnv, int32_t slot);
void queryOnBlock(SMeterQuerySupportObj* pSupporter, int64_t* primaryKeys, int32_t blockStatus, void stableApplyFunctionsOnBlock(STableQuerySupportObj* pSupporter, SMeterDataInfo* pMeterDataInfo,
SBlockInfo* pBlockBasicInfo, SMeterDataInfo* pDataHeadInfoEx, SField* pFields, SBlockInfo* pBlockInfo, SField* pFields, __block_search_fn_t searchFn);
__block_search_fn_t searchFn);
int32_t vnodeFilterQualifiedMeters(SQInfo *pQInfo, int32_t vid, tSidSet *pSidSet, SMeterDataInfo *pMeterDataInfo, int32_t vnodeFilterQualifiedMeters(SQInfo* pQInfo, int32_t vid, tSidSet* pSidSet, SMeterDataInfo* pMeterDataInfo,
int32_t *numOfMeters, SMeterDataInfo ***pReqMeterDataInfo); int32_t* numOfMeters, SMeterDataInfo*** pReqMeterDataInfo);
int32_t vnodeGetVnodeHeaderFileIdx(int32_t* fid, SQueryRuntimeEnv* pRuntimeEnv, int32_t order); int32_t vnodeGetVnodeHeaderFileIndex(int32_t* fid, SQueryRuntimeEnv* pRuntimeEnv, int32_t order);
int32_t createDataBlocksInfoEx(SMeterDataInfo** pMeterDataInfo, int32_t numOfMeters, int32_t createDataBlocksInfoEx(SMeterDataInfo** pMeterDataInfo, int32_t numOfMeters,
SMeterDataBlockInfoEx** pDataBlockInfoEx, int32_t numOfCompBlocks, SMeterDataBlockInfoEx** pDataBlockInfoEx, int32_t numOfCompBlocks,
int32_t* nAllocBlocksInfoSize, int64_t addr); int32_t* nAllocBlocksInfoSize, int64_t addr);
void freeMeterBlockInfoEx(SMeterDataBlockInfoEx* pDataBlockInfoEx, int32_t len); void freeMeterBlockInfoEx(SMeterDataBlockInfoEx* pDataBlockInfoEx, int32_t len);
void setExecutionContext(SMeterQuerySupportObj* pSupporter, SOutputRes* outputRes, int32_t meterIdx, int32_t groupIdx, void setExecutionContext(STableQuerySupportObj* pSupporter, SMeterQueryInfo* pMeterQueryInfo, int32_t meterIdx,
SMeterQueryInfo* sqinfo); int32_t groupIdx, TSKEY nextKey);
int32_t setIntervalQueryExecutionContext(SMeterQuerySupportObj* pSupporter, int32_t meterIdx, SMeterQueryInfo* sqinfo); int32_t setAdditionalInfo(STableQuerySupportObj *pSupporter, int32_t meterIdx, SMeterQueryInfo *pMeterQueryInfo);
void doGetAlignedIntervalQueryRangeImpl(SQuery* pQuery, int64_t pKey, int64_t keyFirst, int64_t keyLast,
int64_t* actualSkey, int64_t* actualEkey, int64_t* skey, int64_t* ekey);
int64_t getQueryStartPositionInCache(SQueryRuntimeEnv* pRuntimeEnv, int32_t* slot, int32_t* pos, bool ignoreQueryRange); int64_t getQueryStartPositionInCache(SQueryRuntimeEnv* pRuntimeEnv, int32_t* slot, int32_t* pos, bool ignoreQueryRange);
int64_t getNextAccessedKeyInData(SQuery* pQuery, int64_t* pPrimaryCol, SBlockInfo* pBlockInfo, int32_t blockStatus);
int32_t getDataBlocksForMeters(SMeterQuerySupportObj* pSupporter, SQuery* pQuery, int32_t numOfMeters, int32_t getDataBlocksForMeters(STableQuerySupportObj* pSupporter, SQuery* pQuery, int32_t numOfMeters,
const char* filePath, SMeterDataInfo** pMeterDataInfo, uint32_t* numOfBlocks); const char* filePath, SMeterDataInfo** pMeterDataInfo, uint32_t* numOfBlocks);
int32_t LoadDatablockOnDemand(SCompBlock* pBlock, SField** pFields, uint8_t* blkStatus, SQueryRuntimeEnv* pRuntimeEnv, int32_t LoadDatablockOnDemand(SCompBlock* pBlock, SField** pFields, uint8_t* blkStatus, SQueryRuntimeEnv* pRuntimeEnv,
int32_t fileIdx, int32_t slotIdx, __block_search_fn_t searchFn, bool onDemand); int32_t fileIdx, int32_t slotIdx, __block_search_fn_t searchFn, bool onDemand);
int32_t vnodeGetHeaderFile(SQueryRuntimeEnv *pRuntimeEnv, int32_t fileIndex); int32_t vnodeGetHeaderFile(SQueryRuntimeEnv* pRuntimeEnv, int32_t fileIndex);
/** /**
* Create SMeterQueryInfo. * Create SMeterQueryInfo.
@ -210,14 +209,14 @@ int32_t vnodeGetHeaderFile(SQueryRuntimeEnv *pRuntimeEnv, int32_t fileIndex);
* @param ekey * @param ekey
* @return * @return
*/ */
SMeterQueryInfo* createMeterQueryInfo(SQuery* pQuery, int32_t sid, TSKEY skey, TSKEY ekey); SMeterQueryInfo* createMeterQueryInfo(STableQuerySupportObj* pSupporter, int32_t sid, TSKEY skey, TSKEY ekey);
/** /**
* Destroy meter query info * Destroy meter query info
* @param pMeterQInfo * @param pMeterQInfo
* @param numOfCols * @param numOfCols
*/ */
void destroyMeterQueryInfo(SMeterQueryInfo *pMeterQueryInfo, int32_t numOfCols); void destroyMeterQueryInfo(SMeterQueryInfo* pMeterQueryInfo, int32_t numOfCols);
/** /**
* change the meter query info for supplement scan * change the meter query info for supplement scan
@ -225,7 +224,8 @@ void destroyMeterQueryInfo(SMeterQueryInfo *pMeterQueryInfo, int32_t numOfCols);
* @param skey * @param skey
* @param ekey * @param ekey
*/ */
void changeMeterQueryInfoForSuppleQuery(SQueryResultBuf* pResultBuf, SMeterQueryInfo *pMeterQueryInfo, TSKEY skey, TSKEY ekey); void changeMeterQueryInfoForSuppleQuery(SQuery* pQuery, SMeterQueryInfo* pMeterQueryInfo,
TSKEY skey, TSKEY ekey);
/** /**
* add the new allocated disk page to meter query info * add the new allocated disk page to meter query info
@ -234,14 +234,8 @@ void changeMeterQueryInfoForSuppleQuery(SQueryResultBuf* pResultBuf, SMeterQuery
* @param pMeterQueryInfo * @param pMeterQueryInfo
* @param pSupporter * @param pSupporter
*/ */
tFilePage* addDataPageForMeterQueryInfo(SQuery* pQuery, SMeterQueryInfo *pMeterQueryInfo, SMeterQuerySupportObj *pSupporter); tFilePage* addDataPageForMeterQueryInfo(SQuery* pQuery, SMeterQueryInfo* pMeterQueryInfo,
STableQuerySupportObj* pSupporter);
/**
* save the query range data into SMeterQueryInfo
* @param pRuntimeEnv
* @param pMeterQueryInfo
*/
void saveIntervalQueryRange(SQueryRuntimeEnv* pRuntimeEnv, SMeterQueryInfo* pMeterQueryInfo);
/** /**
* restore the query range data from SMeterQueryInfo to runtime environment * restore the query range data from SMeterQueryInfo to runtime environment
@ -258,7 +252,7 @@ void restoreIntervalQueryRange(SQueryRuntimeEnv* pRuntimeEnv, SMeterQueryInfo* p
* @param pSupporter * @param pSupporter
* @param key * @param key
*/ */
void setIntervalQueryRange(SMeterQueryInfo *pMeterQueryInfo, SMeterQuerySupportObj* pSupporter, int64_t key); void setIntervalQueryRange(SMeterQueryInfo* pMeterQueryInfo, STableQuerySupportObj* pSupporter, int64_t key);
/** /**
* set the meter data information * set the meter data information
@ -275,16 +269,22 @@ void vnodeCheckIfDataExists(SQueryRuntimeEnv* pRuntimeEnv, SMeterObj* pMeterObj,
void displayInterResult(SData** pdata, SQuery* pQuery, int32_t numOfRows); void displayInterResult(SData** pdata, SQuery* pQuery, int32_t numOfRows);
void vnodePrintQueryStatistics(SMeterQuerySupportObj* pSupporter); void vnodePrintQueryStatistics(STableQuerySupportObj* pSupporter);
void clearGroupResultBuf(SQueryRuntimeEnv *pRuntimeEnv, SOutputRes *pOneOutputRes); void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes);
void copyGroupResultBuf(SQueryRuntimeEnv *pRuntimeEnv, SOutputRes* dst, const SOutputRes* src); void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src);
void resetSlidingWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SSlidingWindowInfo* pSlidingWindowInfo); int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size,
void clearCompletedSlidingWindows(SQueryRuntimeEnv* pRuntimeEnv); int32_t threshold, int16_t type);
int32_t numOfClosedSlidingWindow(SSlidingWindowInfo* pSlidingWindowInfo);
void closeSlidingWindow(SSlidingWindowInfo* pSlidingWindowInfo, int32_t slot); void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv);
void closeAllSlidingWindow(SSlidingWindowInfo* pSlidingWindowInfo); void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo);
void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num);
void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv);
int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo);
void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot);
void closeAllTimeWindow(SWindowResInfo* pWindowResInfo);
#ifdef __cplusplus #ifdef __cplusplus
} }

74
src/system/detail/inc/vnodeRead.h
View File

@ -86,16 +86,26 @@ typedef struct SQueryCostSummary {
} SQueryCostSummary; } SQueryCostSummary;
typedef struct SPosInfo { typedef struct SPosInfo {
int64_t pageId; int16_t pageId;
int32_t rowId; int16_t rowId;
} SPosInfo; } SPosInfo;
typedef struct SOutputRes { typedef struct STimeWindow {
TSKEY skey;
TSKEY ekey;
} STimeWindow;
typedef struct SWindowStatus {
bool closed;
} SWindowStatus;
typedef struct SWindowResult {
uint16_t numOfRows; uint16_t numOfRows;
int32_t nAlloc; SPosInfo pos; // Position of current result in disk-based output buffer
SPosInfo pos; SResultInfo* resultInfo; // For each result column, there is a resultInfo
SResultInfo* resultInfo; STimeWindow window; // The time window that current result covers.
} SOutputRes; SWindowStatus status;
} SWindowResult;
/* /*
* header files info, avoid to iterate the directory, the data is acquired * header files info, avoid to iterate the directory, the data is acquired
@ -118,19 +128,8 @@ typedef struct SQueryFilesInfo {
char dbFilePathPrefix[PATH_MAX]; char dbFilePathPrefix[PATH_MAX];
} SQueryFilesInfo; } SQueryFilesInfo;
typedef struct STimeWindow { typedef struct SWindowResInfo {
TSKEY skey; SWindowResult* pResult; // reference to SQuerySupporter->pResult
TSKEY ekey;
} STimeWindow;
typedef struct SWindowStatus {
STimeWindow window;
bool closed;
} SWindowStatus;
typedef struct SSlidingWindowInfo {
SOutputRes* pResult; // reference to SQuerySupporter->pResult
SWindowStatus* pStatus; // current query window closed or not?
void* hashList; // hash list for quick access void* hashList; // hash list for quick access
int16_t type; // data type for hash key int16_t type; // data type for hash key
int32_t capacity; // max capacity int32_t capacity; // max capacity
@ -140,7 +139,7 @@ typedef struct SSlidingWindowInfo {
int64_t startTime; // start time of the first time window for sliding query int64_t startTime; // start time of the first time window for sliding query
int64_t prevSKey; // previous (not completed) sliding window start key int64_t prevSKey; // previous (not completed) sliding window start key
int64_t threshold; // threshold for return completed results. int64_t threshold; // threshold for return completed results.
} SSlidingWindowInfo; } SWindowResInfo;
typedef struct SQueryRuntimeEnv { typedef struct SQueryRuntimeEnv {
SPositionInfo startPos; /* the start position, used for secondary/third iteration */ SPositionInfo startPos; /* the start position, used for secondary/third iteration */
@ -161,28 +160,25 @@ typedef struct SQueryRuntimeEnv {
SQueryFilesInfo vnodeFileInfo; SQueryFilesInfo vnodeFileInfo;
int16_t numOfRowsPerPage; int16_t numOfRowsPerPage;
int16_t offset[TSDB_MAX_COLUMNS]; int16_t offset[TSDB_MAX_COLUMNS];
int16_t scanFlag; // denotes reversed scan of data or not uint16_t scanFlag; // denotes reversed scan of data or not
SInterpolationInfo interpoInfo; SInterpolationInfo interpoInfo;
SData** pInterpoBuf; SData** pInterpoBuf;
SSlidingWindowInfo swindowResInfo; SWindowResInfo windowResInfo;
STSBuf* pTSBuf; STSBuf* pTSBuf;
STSCursor cur; STSCursor cur;
SQueryCostSummary summary; SQueryCostSummary summary;
bool stableQuery; // is super table query or not
STimeWindow intervalWindow; // the complete time window, not affected by the actual data distribution SQueryDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file
/* /*
* Temporarily hold the in-memory cache block info during scan cache blocks * Temporarily hold the in-memory cache block info during scan cache blocks
* Here we do not use the cacheblock info from pMeterObj, simple because it may change anytime * Here we do not use the cache block info from pMeterObj, simple because it may change anytime
* during the query by the subumit/insert handling threads. * during the query by the submit/insert handling threads.
* So we keep a copy of the support structure as well as the cache block data itself. * So we keep a copy of the support structure as well as the cache block data itself.
*/ */
SCacheBlock cacheBlock; SCacheBlock cacheBlock;
SQueryResultBuf* pResultBuf;
bool stableQuery; // is super table query or not
} SQueryRuntimeEnv; } SQueryRuntimeEnv;
/* intermediate pos during multimeter query involves interval */ /* intermediate pos during multimeter query involves interval */
@ -191,14 +187,12 @@ typedef struct SMeterQueryInfo {
int64_t skey; int64_t skey;
int64_t ekey; int64_t ekey;
int32_t numOfRes; int32_t numOfRes;
int32_t reverseIndex; // reversed output indicator, start from (numOfRes-1)
int16_t reverseFillRes; // denote if reverse fill the results in supplementary scan required or not
int16_t queryRangeSet; // denote if the query range is set, only available for interval query int16_t queryRangeSet; // denote if the query range is set, only available for interval query
int16_t lastResRows;
int64_t tag; int64_t tag;
STSCursor cur; STSCursor cur;
SResultInfo* resultInfo;
int32_t sid; // for retrieve the page id list int32_t sid; // for retrieve the page id list
SWindowResInfo windowResInfo;
} SMeterQueryInfo; } SMeterQueryInfo;
typedef struct SMeterDataInfo { typedef struct SMeterDataInfo {
@ -212,7 +206,7 @@ typedef struct SMeterDataInfo {
SMeterQueryInfo* pMeterQInfo; SMeterQueryInfo* pMeterQInfo;
} SMeterDataInfo; } SMeterDataInfo;
typedef struct SMeterQuerySupportObj { typedef struct STableQuerySupportObj {
void* pMetersHashTable; // meter table hash list void* pMetersHashTable; // meter table hash list
SMeterSidExtInfo** pMeterSidExtInfo; SMeterSidExtInfo** pMeterSidExtInfo;
@ -225,13 +219,11 @@ typedef struct SMeterQuerySupportObj {
* rows may be generated by a specific subgroup. When query on all subgroups is executed, * rows may be generated by a specific subgroup. When query on all subgroups is executed,
* the result is copy to output buffer. This attribution is not used during single meter query processing. * the result is copy to output buffer. This attribution is not used during single meter query processing.
*/ */
SOutputRes* pResult;
SQueryRuntimeEnv runtimeEnv; SQueryRuntimeEnv runtimeEnv;
int64_t rawSKey; int64_t rawSKey;
int64_t rawEKey; int64_t rawEKey;
int32_t subgroupIdx; int32_t subgroupIdx;
int32_t offset; /* offset in group result set of subgroup */ int32_t offset; /* offset in group result set of subgroup */
tSidSet* pSidSet; tSidSet* pSidSet;
/* /*
@ -247,7 +239,7 @@ typedef struct SMeterQuerySupportObj {
SMeterDataInfo* pMeterDataInfo; SMeterDataInfo* pMeterDataInfo;
TSKEY* tsList; TSKEY* tsList;
} SMeterQuerySupportObj; } STableQuerySupportObj;
typedef struct _qinfo { typedef struct _qinfo {
uint64_t signature; uint64_t signature;
@ -273,18 +265,18 @@ typedef struct _qinfo {
SMeterObj* pObj; SMeterObj* pObj;
sem_t dataReady; sem_t dataReady;
SMeterQuerySupportObj* pMeterQuerySupporter; STableQuerySupportObj* pTableQuerySupporter;
int (*fp)(SMeterObj*, SQuery*); int (*fp)(SMeterObj*, SQuery*);
} SQInfo; } SQInfo;
int32_t vnodeQuerySingleMeterPrepare(SQInfo* pQInfo, SMeterObj* pMeterObj, SMeterQuerySupportObj* pSMultiMeterObj, int32_t vnodeQueryTablePrepare(SQInfo* pQInfo, SMeterObj* pMeterObj, STableQuerySupportObj* pSMultiMeterObj,
void* param); void* param);
void vnodeQueryFreeQInfoEx(SQInfo* pQInfo); void vnodeQueryFreeQInfoEx(SQInfo* pQInfo);
bool vnodeParametersSafetyCheck(SQuery* pQuery); bool vnodeParametersSafetyCheck(SQuery* pQuery);
int32_t vnodeMultiMeterQueryPrepare(SQInfo* pQInfo, SQuery* pQuery, void* param); int32_t vnodeSTableQueryPrepare(SQInfo* pQInfo, SQuery* pQuery, void* param);
/** /**
* decrease the numofQuery of each table that is queried, enable the * decrease the numofQuery of each table that is queried, enable the

3383
src/system/detail/src/vnodeQueryImpl.c
View File

File diff suppressed because it is too large Load Diff

474
src/system/detail/src/vnodeQueryProcess.c
View File

@ -26,8 +26,9 @@
#include "vnodeQueryImpl.h" #include "vnodeQueryImpl.h"
#define ALL_CACHE_BLOCKS_CHECKED(q) \ #define ALL_CACHE_BLOCKS_CHECKED(q) \
(((q)->slot == (q)->currentSlot && QUERY_IS_ASC_QUERY(q)) || ((q)->slot == (q)->firstSlot && (!QUERY_IS_ASC_QUERY(q)))) (((q)->slot == (q)->currentSlot && QUERY_IS_ASC_QUERY(q)) || \
((q)->slot == (q)->firstSlot && (!QUERY_IS_ASC_QUERY(q))))
#define FORWARD_CACHE_BLOCK_CHECK_SLOT(slot, step, maxblocks) (slot) = ((slot) + (step) + (maxblocks)) % (maxblocks); #define FORWARD_CACHE_BLOCK_CHECK_SLOT(slot, step, maxblocks) (slot) = ((slot) + (step) + (maxblocks)) % (maxblocks);
@ -47,23 +48,10 @@ static bool isGroupbyEachTable(SSqlGroupbyExpr *pGroupbyExpr, tSidSet *pSidset)
return false; return false;
} }
static bool doCheckWithPrevQueryRange(SQInfo *pQInfo, TSKEY nextKey, SMeterDataInfo *pMeterInfo) { static bool doCheckWithPrevQueryRange(SQuery *pQuery, TSKEY nextKey) {
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter;
SQuery * pQuery = &pQInfo->query;
SMeterObj * pMeterObj = pMeterInfo->pMeterObj;
/* no data for current query */
if ((nextKey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || if ((nextKey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(nextKey < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) { (nextKey < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
if (((nextKey > pSupporter->rawEKey) && QUERY_IS_ASC_QUERY(pQuery)) || return false;
((nextKey < pSupporter->rawEKey) && (!QUERY_IS_ASC_QUERY(pQuery)))) {
dTrace("QInfo:%p vid:%d sid:%d id:%s, no data qualified in block, ignore", pQInfo, pMeterObj->vnode,
pMeterObj->sid, pMeterObj->meterId);
return false;
} else { // in case of interval query, forward the query range
setIntervalQueryRange(pMeterInfo->pMeterQInfo, pSupporter, nextKey);
}
} }
return true; return true;
@ -86,8 +74,8 @@ static void setStartPositionForCacheBlock(SQuery *pQuery, SCacheBlock *pBlock, b
} }
static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) {
SQuery* pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) {
SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]);
if (pResInfo != NULL) { if (pResInfo != NULL) {
@ -96,10 +84,10 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) {
} }
} }
static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterInfo) { static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo) {
SQuery * pQuery = &pQInfo->query; SQuery * pQuery = &pQInfo->query;
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->pMeterQuerySupporter->runtimeEnv; SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->pTableQuerySupporter->runtimeEnv;
SMeterSidExtInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo; SMeterSidExtInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo;
@ -131,17 +119,18 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterInfo) {
pQInfo->pObj = pMeterObj; pQInfo->pObj = pMeterObj;
pRuntimeEnv->pMeterObj = pMeterObj; pRuntimeEnv->pMeterObj = pMeterObj;
if (pMeterInfo[k].pMeterQInfo == NULL) { if (pMeterDataInfo[k].pMeterQInfo == NULL) {
pMeterInfo[k].pMeterQInfo = createMeterQueryInfo(pQuery, pMeterObj->sid, pSupporter->rawSKey, pSupporter->rawEKey); pMeterDataInfo[k].pMeterQInfo =
createMeterQueryInfo(pSupporter, pMeterObj->sid, pSupporter->rawSKey, pSupporter->rawEKey);
} }
if (pMeterInfo[k].pMeterObj == NULL) { // no data in disk for this meter, set its pointer if (pMeterDataInfo[k].pMeterObj == NULL) { // no data in disk for this meter, set its pointer
setMeterDataInfo(&pMeterInfo[k], pMeterObj, k, groupIdx); setMeterDataInfo(&pMeterDataInfo[k], pMeterObj, k, groupIdx);
} }
assert(pMeterInfo[k].meterOrderIdx == k && pMeterObj == pMeterInfo[k].pMeterObj); assert(pMeterDataInfo[k].meterOrderIdx == k && pMeterObj == pMeterDataInfo[k].pMeterObj);
SMeterQueryInfo *pMeterQueryInfo = pMeterInfo[k].pMeterQInfo; SMeterQueryInfo *pMeterQueryInfo = pMeterDataInfo[k].pMeterQInfo;
restoreIntervalQueryRange(pRuntimeEnv, pMeterQueryInfo); restoreIntervalQueryRange(pRuntimeEnv, pMeterQueryInfo);
/* /*
@ -154,29 +143,18 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterInfo) {
vnodeUpdateQueryColumnIndex(pQuery, pMeterObj); vnodeUpdateQueryColumnIndex(pQuery, pMeterObj);
vnodeUpdateFilterColumnIndex(pQuery); vnodeUpdateFilterColumnIndex(pQuery);
if (pQuery->nAggTimeInterval == 0) { if ((pQuery->lastKey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
if ((pQuery->lastKey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || (pQuery->lastKey < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
(pQuery->lastKey < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) { dTrace("QInfo:%p vid:%d sid:%d id:%s, query completed, ignore data in cache. qrange:%" PRId64 "-%" PRId64
dTrace( ", lastKey:%" PRId64,
"QInfo:%p vid:%d sid:%d id:%s, query completed, ignore data in cache. qrange:%" PRId64 "-%" PRId64 ", " pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey,
"lastKey:%" PRId64, pQuery->lastKey);
pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey,
pQuery->lastKey);
continue; continue;
}
setExecutionContext(pSupporter, pSupporter->pResult, k, pMeterInfo[k].groupIdx, pMeterQueryInfo);
} else {
int32_t ret = setIntervalQueryExecutionContext(pSupporter, k, pMeterQueryInfo);
if (ret != TSDB_CODE_SUCCESS) {
pQInfo->killed = 1;
return;
}
} }
qTrace("QInfo:%p vid:%d sid:%d id:%s, query in cache, qrange:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64, pQInfo, pMeterObj->vnode, qTrace("QInfo:%p vid:%d sid:%d id:%s, query in cache, qrange:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64, pQInfo,
pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, pQuery->lastKey); pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, pQuery->lastKey);
/* /*
* find the appropriated start position in cache * find the appropriated start position in cache
@ -186,7 +164,7 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterInfo) {
* should be ignored (the fourth parameter). * should be ignored (the fourth parameter).
*/ */
TSKEY nextKey = getQueryStartPositionInCache(pRuntimeEnv, &pQuery->slot, &pQuery->pos, true); TSKEY nextKey = getQueryStartPositionInCache(pRuntimeEnv, &pQuery->slot, &pQuery->pos, true);
if (nextKey < 0) { if (nextKey < 0 || !doCheckWithPrevQueryRange(pQuery, nextKey)) {
qTrace("QInfo:%p vid:%d sid:%d id:%s, no data qualified in cache, cache blocks:%d, lastKey:%" PRId64, pQInfo, qTrace("QInfo:%p vid:%d sid:%d id:%s, no data qualified in cache, cache blocks:%d, lastKey:%" PRId64, pQInfo,
pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->numOfBlocks, pQuery->lastKey); pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->numOfBlocks, pQuery->lastKey);
continue; continue;
@ -199,10 +177,6 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterInfo) {
continue; continue;
} }
if (!doCheckWithPrevQueryRange(pQInfo, nextKey, &pMeterInfo[k])) {
continue;
}
bool firstCheckSlot = true; bool firstCheckSlot = true;
SCacheInfo *pCacheInfo = (SCacheInfo *)pMeterObj->pCache; SCacheInfo *pCacheInfo = (SCacheInfo *)pMeterObj->pCache;
@ -224,24 +198,39 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterInfo) {
setStartPositionForCacheBlock(pQuery, pBlock, &firstCheckSlot); setStartPositionForCacheBlock(pQuery, pBlock, &firstCheckSlot);
TSKEY* primaryKeys = (TSKEY*) pRuntimeEnv->primaryColBuffer->data; TSKEY *primaryKeys = (TSKEY *)pRuntimeEnv->primaryColBuffer->data;
TSKEY key = primaryKeys[pQuery->pos];
// in handling file data block, the timestamp range validation is done during fetching candidate file blocks // in handling file data block, the timestamp range validation is done during fetching candidate file blocks
if ((primaryKeys[pQuery->pos] > pSupporter->rawEKey && QUERY_IS_ASC_QUERY(pQuery)) || if ((key > pSupporter->rawEKey && QUERY_IS_ASC_QUERY(pQuery)) ||
(primaryKeys[pQuery->pos] < pSupporter->rawEKey && !QUERY_IS_ASC_QUERY(pQuery))) { (key < pSupporter->rawEKey && !QUERY_IS_ASC_QUERY(pQuery))) {
break; break;
} }
if (pQuery->intervalTime == 0) {
setExecutionContext(pSupporter, pMeterQueryInfo, k, pMeterDataInfo[k].groupIdx, key);
} else {
setIntervalQueryRange(pMeterQueryInfo, pSupporter, key);
int32_t ret = setAdditionalInfo(pSupporter, k, pMeterQueryInfo);
if (ret != TSDB_CODE_SUCCESS) {
pQInfo->killed = 1;
return;
}
}
qTrace("QInfo:%p vid:%d sid:%d id:%s, query in cache, qrange:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64, pQInfo,
pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, pQuery->lastKey);
// only record the key on last block // only record the key on last block
SET_CACHE_BLOCK_FLAG(pRuntimeEnv->blockStatus); SET_CACHE_BLOCK_FLAG(pRuntimeEnv->blockStatus);
SBlockInfo binfo = getBlockBasicInfo(pRuntimeEnv, pBlock, BLK_CACHE_BLOCK); SBlockInfo binfo = getBlockInfo(pRuntimeEnv);
dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", fileId:%d, slot:%d, pos:%d, bstatus:%d", dTrace("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", fileId:%d, slot:%d, pos:%d, bstatus:%d",
GET_QINFO_ADDR(pQuery), binfo.keyFirst, binfo.keyLast, pQuery->fileId, pQuery->slot, pQuery->pos, GET_QINFO_ADDR(pQuery), binfo.keyFirst, binfo.keyLast, pQuery->fileId, pQuery->slot, pQuery->pos,
pRuntimeEnv->blockStatus); pRuntimeEnv->blockStatus);
totalBlocks++; totalBlocks++;
queryOnBlock(pSupporter, primaryKeys, pRuntimeEnv->blockStatus, &binfo, &pMeterInfo[k], NULL, searchFn); stableApplyFunctionsOnBlock(pSupporter, &pMeterDataInfo[k], &binfo, NULL, searchFn);
if (ALL_CACHE_BLOCKS_CHECKED(pQuery)) { if (ALL_CACHE_BLOCKS_CHECKED(pQuery)) {
break; break;
@ -266,7 +255,7 @@ static void queryOnMultiDataCache(SQInfo *pQInfo, SMeterDataInfo *pMeterInfo) {
static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo) { static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo) {
SQuery * pQuery = &pQInfo->query; SQuery * pQuery = &pQInfo->query;
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv; SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv;
SMeterDataBlockInfoEx *pDataBlockInfoEx = NULL; SMeterDataBlockInfoEx *pDataBlockInfoEx = NULL;
int32_t nAllocBlocksInfoSize = 0; int32_t nAllocBlocksInfoSize = 0;
@ -274,9 +263,9 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
SMeterObj * pTempMeter = getMeterObj(pSupporter->pMetersHashTable, pSupporter->pMeterSidExtInfo[0]->sid); SMeterObj * pTempMeter = getMeterObj(pSupporter->pMetersHashTable, pSupporter->pMeterSidExtInfo[0]->sid);
__block_search_fn_t searchFn = vnodeSearchKeyFunc[pTempMeter->searchAlgorithm]; __block_search_fn_t searchFn = vnodeSearchKeyFunc[pTempMeter->searchAlgorithm];
int32_t vnodeId = pTempMeter->vnode; int32_t vnodeId = pTempMeter->vnode;
SQueryFilesInfo* pVnodeFileInfo = &pRuntimeEnv->vnodeFileInfo; SQueryFilesInfo *pVnodeFileInfo = &pRuntimeEnv->vnodeFileInfo;
dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pVnodeFileInfo->numOfFiles); dTrace("QInfo:%p start to check data blocks in %d files", pQInfo, pVnodeFileInfo->numOfFiles);
int32_t fid = QUERY_IS_ASC_QUERY(pQuery) ? -1 : INT32_MAX; int32_t fid = QUERY_IS_ASC_QUERY(pQuery) ? -1 : INT32_MAX;
@ -291,7 +280,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
break; break;
} }
int32_t fileIdx = vnodeGetVnodeHeaderFileIdx(&fid, pRuntimeEnv, pQuery->order.order); int32_t fileIdx = vnodeGetVnodeHeaderFileIndex(&fid, pRuntimeEnv, pQuery->order.order);
if (fileIdx < 0) { // no valid file, abort current search if (fileIdx < 0) { // no valid file, abort current search
break; break;
} }
@ -304,25 +293,25 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
fid += step; fid += step;
continue; continue;
} }
int32_t numOfQualifiedMeters = 0; int32_t numOfQualifiedMeters = 0;
assert(fileIdx == pRuntimeEnv->vnodeFileInfo.current); assert(fileIdx == pRuntimeEnv->vnodeFileInfo.current);
SMeterDataInfo **pReqMeterDataInfo = NULL; SMeterDataInfo **pReqMeterDataInfo = NULL;
int32_t ret = vnodeFilterQualifiedMeters(pQInfo, vnodeId, pSupporter->pSidSet, pMeterDataInfo, int32_t ret = vnodeFilterQualifiedMeters(pQInfo, vnodeId, pSupporter->pSidSet, pMeterDataInfo,
&numOfQualifiedMeters, &pReqMeterDataInfo); &numOfQualifiedMeters, &pReqMeterDataInfo);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
dError("QInfo:%p failed to create meterdata struct to perform query processing, abort", pQInfo); dError("QInfo:%p failed to create meterdata struct to perform query processing, abort", pQInfo);
tfree(pReqMeterDataInfo); tfree(pReqMeterDataInfo);
pQInfo->code = -ret; pQInfo->code = -ret;
pQInfo->killed = 1; pQInfo->killed = 1;
return; return;
} }
dTrace("QInfo:%p file:%s, %d meters qualified", pQInfo, pVnodeFileInfo->dataFilePath, numOfQualifiedMeters); dTrace("QInfo:%p file:%s, %d meters qualified", pQInfo, pVnodeFileInfo->dataFilePath, numOfQualifiedMeters);
// none of meters in query set have pHeaderFileData in this file, try next file // none of meters in query set have pHeaderFileData in this file, try next file
if (numOfQualifiedMeters == 0) { if (numOfQualifiedMeters == 0) {
fid += step; fid += step;
@ -335,17 +324,17 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
pReqMeterDataInfo, &numOfBlocks); pReqMeterDataInfo, &numOfBlocks);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
dError("QInfo:%p failed to get data block before scan data blocks, abort", pQInfo); dError("QInfo:%p failed to get data block before scan data blocks, abort", pQInfo);
tfree(pReqMeterDataInfo); tfree(pReqMeterDataInfo);
pQInfo->code = -ret; pQInfo->code = -ret;
pQInfo->killed = 1; pQInfo->killed = 1;
return; return;
} }
dTrace("QInfo:%p file:%s, %d meters contains %d blocks to be checked", pQInfo, pVnodeFileInfo->dataFilePath, dTrace("QInfo:%p file:%s, %d meters contains %d blocks to be checked", pQInfo, pVnodeFileInfo->dataFilePath,
numOfQualifiedMeters, numOfBlocks); numOfQualifiedMeters, numOfBlocks);
if (numOfBlocks == 0) { if (numOfBlocks == 0) {
fid += step; fid += step;
tfree(pReqMeterDataInfo); tfree(pReqMeterDataInfo);
@ -353,7 +342,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
} }
ret = createDataBlocksInfoEx(pReqMeterDataInfo, numOfQualifiedMeters, &pDataBlockInfoEx, numOfBlocks, ret = createDataBlocksInfoEx(pReqMeterDataInfo, numOfQualifiedMeters, &pDataBlockInfoEx, numOfBlocks,
&nAllocBlocksInfoSize, (int64_t)pQInfo); &nAllocBlocksInfoSize, (int64_t)pQInfo);
if (ret != TSDB_CODE_SUCCESS) { // failed to create data blocks if (ret != TSDB_CODE_SUCCESS) { // failed to create data blocks
dError("QInfo:%p build blockInfoEx failed, abort", pQInfo); dError("QInfo:%p build blockInfoEx failed, abort", pQInfo);
tfree(pReqMeterDataInfo); tfree(pReqMeterDataInfo);
@ -383,8 +372,8 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
stimeUnit = taosGetTimestampMs(); stimeUnit = taosGetTimestampMs();
} else if ((j % TRACE_OUTPUT_BLOCK_CNT) == 0) { } else if ((j % TRACE_OUTPUT_BLOCK_CNT) == 0) {
etimeUnit = taosGetTimestampMs(); etimeUnit = taosGetTimestampMs();
dTrace("QInfo:%p load and check %" PRId64 " blocks, and continue. elapsed:%" PRId64 " ms", pQInfo, TRACE_OUTPUT_BLOCK_CNT, dTrace("QInfo:%p load and check %" PRId64 " blocks, and continue. elapsed:%" PRId64 " ms", pQInfo,
etimeUnit - stimeUnit); TRACE_OUTPUT_BLOCK_CNT, etimeUnit - stimeUnit);
stimeUnit = taosGetTimestampMs(); stimeUnit = taosGetTimestampMs();
} }
@ -398,22 +387,52 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
restoreIntervalQueryRange(pRuntimeEnv, pMeterQueryInfo); restoreIntervalQueryRange(pRuntimeEnv, pMeterQueryInfo);
if (pQuery->nAggTimeInterval == 0 && !isSumAvgRateQuery(pQuery)) { // normal query if ((pQuery->lastKey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
if ((pQuery->lastKey > pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) || (pQuery->lastKey < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
(pQuery->lastKey < pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery))) { qTrace("QInfo:%p vid:%d sid:%d id:%s, query completed, no need to scan this data block. qrange:%" PRId64
qTrace( "-%" PRId64 ", lastKey:%" PRId64,
"QInfo:%p vid:%d sid:%d id:%s, query completed, no need to scan this data block. qrange:%" PRId64 "-%" PRId64 ", " pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey,
"lastKey:%" PRId64, pQuery->lastKey);
pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey,
pQuery->lastKey);
continue;
}
SCompBlock *pBlock = pInfoEx->pBlock.compBlock;
bool ondemandLoad = onDemandLoadDatablock(pQuery, pMeterQueryInfo->queryRangeSet);
ret = LoadDatablockOnDemand(pBlock, &pInfoEx->pBlock.fields, &pRuntimeEnv->blockStatus, pRuntimeEnv, fileIdx,
pInfoEx->blockIndex, searchFn, ondemandLoad);
if (ret != DISK_DATA_LOADED) {
pSummary->skippedFileBlocks++;
continue;
}
SBlockInfo binfo = getBlockBasicInfo(pRuntimeEnv, pBlock, BLK_FILE_BLOCK);
int64_t nextKey = -1;
assert(pQuery->pos >= 0 && pQuery->pos < pBlock->numOfPoints);
TSKEY *primaryKeys = (TSKEY *)pRuntimeEnv->primaryColBuffer->data;
if (IS_DATA_BLOCK_LOADED(pRuntimeEnv->blockStatus) && needPrimaryTimestampCol(pQuery, &binfo)) {
nextKey = primaryKeys[pQuery->pos];
if (!doCheckWithPrevQueryRange(pQuery, nextKey)) {
qTrace("QInfo:%p vid:%d sid:%d id:%s, no data qualified in data file, lastKey:%" PRId64, pQInfo,
pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->numOfBlocks, pQuery->lastKey);
continue; continue;
} }
} else {
// if data block is not loaded, it must be the intermediate blocks
assert((pBlock->keyFirst >= pQuery->lastKey && pBlock->keyLast <= pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(pBlock->keyFirst >= pQuery->ekey && pBlock->keyLast <= pQuery->lastKey && !QUERY_IS_ASC_QUERY(pQuery)));
nextKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlock->keyFirst : pBlock->keyLast;
}
setExecutionContext(pSupporter, pSupporter->pResult, pOneMeterDataInfo->meterOrderIdx, if (pQuery->intervalTime == 0) {
pOneMeterDataInfo->groupIdx, pMeterQueryInfo); setExecutionContext(pSupporter, pMeterQueryInfo, pOneMeterDataInfo->meterOrderIdx, pOneMeterDataInfo->groupIdx,
nextKey);
} else { // interval query } else { // interval query
ret = setIntervalQueryExecutionContext(pSupporter, pOneMeterDataInfo->meterOrderIdx, pMeterQueryInfo); setIntervalQueryRange(pMeterQueryInfo, pSupporter, nextKey);
ret = setAdditionalInfo(pSupporter, pOneMeterDataInfo->meterOrderIdx, pMeterQueryInfo);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
tfree(pReqMeterDataInfo); // error code has been set tfree(pReqMeterDataInfo); // error code has been set
pQInfo->killed = 1; pQInfo->killed = 1;
@ -421,33 +440,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
} }
} }
SCompBlock *pBlock = pInfoEx->pBlock.compBlock; stableApplyFunctionsOnBlock(pSupporter, pOneMeterDataInfo, &binfo, pInfoEx->pBlock.fields, searchFn);
bool ondemandLoad = onDemandLoadDatablock(pQuery, pMeterQueryInfo->queryRangeSet);
int32_t ret = LoadDatablockOnDemand(pBlock, &pInfoEx->pBlock.fields, &pRuntimeEnv->blockStatus, pRuntimeEnv,
fileIdx, pInfoEx->blockIndex, searchFn, ondemandLoad);
if (ret != DISK_DATA_LOADED) {
pSummary->skippedFileBlocks++;
continue;
}
SBlockInfo binfo = getBlockBasicInfo(pRuntimeEnv, pBlock, BLK_FILE_BLOCK);
assert(pQuery->pos >= 0 && pQuery->pos < pBlock->numOfPoints);
TSKEY *primaryKeys = (TSKEY *)pRuntimeEnv->primaryColBuffer->data;
if (IS_DATA_BLOCK_LOADED(pRuntimeEnv->blockStatus) && needPrimaryTimestampCol(pQuery, &binfo)) {
TSKEY nextKey = primaryKeys[pQuery->pos];
if (!doCheckWithPrevQueryRange(pQInfo, nextKey, pOneMeterDataInfo)) {
continue;
}
} else {
// if data block is not loaded, it must be the intermediate blocks
assert((pBlock->keyFirst >= pQuery->lastKey && pBlock->keyLast <= pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(pBlock->keyFirst >= pQuery->ekey && pBlock->keyLast <= pQuery->lastKey && !QUERY_IS_ASC_QUERY(pQuery)));
}
queryOnBlock(pSupporter, primaryKeys, pRuntimeEnv->blockStatus, &binfo, pOneMeterDataInfo, pInfoEx->pBlock.fields,
searchFn);
} }
tfree(pReqMeterDataInfo); tfree(pReqMeterDataInfo);
@ -470,7 +463,7 @@ static void queryOnMultiDataFiles(SQInfo *pQInfo, SMeterDataInfo *pMeterDataInfo
static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *dataInCache, int32_t index, static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *dataInCache, int32_t index,
int32_t start) { int32_t start) {
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SMeterSidExtInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo; SMeterSidExtInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo;
SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv; SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv;
@ -486,13 +479,13 @@ static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *
vnodeSetTagValueInParam(pSupporter->pSidSet, pRuntimeEnv, pMeterSidExtInfo[index]); vnodeSetTagValueInParam(pSupporter->pSidSet, pRuntimeEnv, pMeterSidExtInfo[index]);
dTrace("QInfo:%p query on (%d): vid:%d sid:%d meterId:%s, qrange:%" PRId64 "-%" PRId64, pQInfo, index - start, pMeterObj->vnode, dTrace("QInfo:%p query on (%d): vid:%d sid:%d meterId:%s, qrange:%" PRId64 "-%" PRId64, pQInfo, index - start,
pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey); pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey);
pQInfo->pObj = pMeterObj; pQInfo->pObj = pMeterObj;
pQuery->lastKey = pQuery->skey; pQuery->lastKey = pQuery->skey;
pRuntimeEnv->pMeterObj = pMeterObj; pRuntimeEnv->pMeterObj = pMeterObj;
vnodeUpdateQueryColumnIndex(pQuery, pRuntimeEnv->pMeterObj); vnodeUpdateQueryColumnIndex(pQuery, pRuntimeEnv->pMeterObj);
vnodeUpdateFilterColumnIndex(pQuery); vnodeUpdateFilterColumnIndex(pQuery);
@ -500,8 +493,8 @@ static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *
// data in file or cache is not qualified for the query. abort // data in file or cache is not qualified for the query. abort
if (!(dataInCache || dataInDisk)) { if (!(dataInCache || dataInDisk)) {
dTrace("QInfo:%p vid:%d sid:%d meterId:%s, qrange:%" PRId64 "-%" PRId64 ", nores, %p", pQInfo, pMeterObj->vnode, pMeterObj->sid, dTrace("QInfo:%p vid:%d sid:%d meterId:%s, qrange:%" PRId64 "-%" PRId64 ", nores, %p", pQInfo, pMeterObj->vnode,
pMeterObj->meterId, pQuery->skey, pQuery->ekey, pQuery); pMeterObj->sid, pMeterObj->meterId, pQuery->skey, pQuery->ekey, pQuery);
return false; return false;
} }
@ -525,7 +518,7 @@ static bool multimeterMultioutputHelper(SQInfo *pQInfo, bool *dataInDisk, bool *
static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start) { static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start) {
SQuery * pQuery = &pQInfo->query; SQuery * pQuery = &pQInfo->query;
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv; SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv;
bool dataInDisk = true; bool dataInDisk = true;
@ -556,7 +549,7 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start
pointInterpSupporterDestroy(&pointInterpSupporter); pointInterpSupporterDestroy(&pointInterpSupporter);
vnodeScanAllData(pRuntimeEnv); vnodeScanAllData(pRuntimeEnv);
// first/last_row query, do not invoke the finalize for super table query // first/last_row query, do not invoke the finalize for super table query
doFinalizeResult(pRuntimeEnv); doFinalizeResult(pRuntimeEnv);
@ -580,7 +573,7 @@ static int64_t doCheckMetersInGroup(SQInfo *pQInfo, int32_t index, int32_t start
* @param pQInfo * @param pQInfo
*/ */
static void vnodeSTableSeqProcessor(SQInfo *pQInfo) { static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SMeterSidExtInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo; SMeterSidExtInfo **pMeterSidExtInfo = pSupporter->pMeterSidExtInfo;
SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv; SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv;
@ -589,10 +582,10 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
tSidSet *pSids = pSupporter->pSidSet; tSidSet *pSids = pSupporter->pSidSet;
int32_t vid = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[0]->sid)->vnode; int32_t vid = getMeterObj(pSupporter->pMetersHashTable, pMeterSidExtInfo[0]->sid)->vnode;
if (isPointInterpoQuery(pQuery)) { if (isPointInterpoQuery(pQuery)) {
resetCtxOutputBuf(pRuntimeEnv); resetCtxOutputBuf(pRuntimeEnv);
assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0);
while (pSupporter->subgroupIdx < pSids->numOfSubSet) { while (pSupporter->subgroupIdx < pSids->numOfSubSet) {
@ -600,8 +593,8 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
int32_t end = pSids->starterPos[pSupporter->subgroupIdx + 1] - 1; int32_t end = pSids->starterPos[pSupporter->subgroupIdx + 1] - 1;
if (isFirstLastRowQuery(pQuery)) { if (isFirstLastRowQuery(pQuery)) {
dTrace("QInfo:%p last_row query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, dTrace("QInfo:%p last_row query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, pSids->numOfSubSet,
pSids->numOfSubSet, pSupporter->subgroupIdx); pSupporter->subgroupIdx);
TSKEY key = -1; TSKEY key = -1;
int32_t index = -1; int32_t index = -1;
@ -633,8 +626,8 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
int64_t num = doCheckMetersInGroup(pQInfo, index, start); int64_t num = doCheckMetersInGroup(pQInfo, index, start);
assert(num >= 0); assert(num >= 0);
} else { } else {
dTrace("QInfo:%p interp query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, dTrace("QInfo:%p interp query on vid:%d, numOfGroups:%d, current group:%d", pQInfo, vid, pSids->numOfSubSet,
pSids->numOfSubSet, pSupporter->subgroupIdx); pSupporter->subgroupIdx);
for (int32_t k = start; k <= end; ++k) { for (int32_t k = start; k <= end; ++k) {
if (isQueryKilled(pQuery)) { if (isQueryKilled(pQuery)) {
@ -670,7 +663,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
* we need to return it to client in the first place. * we need to return it to client in the first place.
*/ */
if (pSupporter->subgroupIdx > 0) { if (pSupporter->subgroupIdx > 0) {
copyFromGroupBuf(pQInfo, pSupporter->pResult); copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
pQInfo->pointsRead += pQuery->pointsRead; pQInfo->pointsRead += pQuery->pointsRead;
if (pQuery->pointsRead > 0) { if (pQuery->pointsRead > 0) {
@ -681,10 +674,10 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
if (pSupporter->meterIdx >= pSids->numOfSids) { if (pSupporter->meterIdx >= pSids->numOfSids) {
return; return;
} }
resetCtxOutputBuf(pRuntimeEnv); resetCtxOutputBuf(pRuntimeEnv);
resetSlidingWindowInfo(pRuntimeEnv, &pRuntimeEnv->swindowResInfo); resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo);
while (pSupporter->meterIdx < pSupporter->numOfMeters) { while (pSupporter->meterIdx < pSupporter->numOfMeters) {
int32_t k = pSupporter->meterIdx; int32_t k = pSupporter->meterIdx;
@ -692,9 +685,8 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK); setQueryStatus(pQuery, QUERY_NO_DATA_TO_CHECK);
return; return;
} }
TSKEY skey = pQInfo->pMeterQuerySupporter->pMeterSidExtInfo[k]->key; TSKEY skey = pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key;
if (skey > 0) { if (skey > 0) {
pQuery->skey = skey; pQuery->skey = skey;
} }
@ -747,10 +739,10 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
pSupporter->meterIdx = pSupporter->pSidSet->numOfSids; pSupporter->meterIdx = pSupporter->pSidSet->numOfSids;
break; break;
} }
// enable execution for next table, when handling the projection query // enable execution for next table, when handling the projection query
enableExecutionForNextTable(pRuntimeEnv); enableExecutionForNextTable(pRuntimeEnv);
if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) {
/* /*
* query range is identical in terms of all meters involved in query, * query range is identical in terms of all meters involved in query,
@ -762,7 +754,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
pQuery->ekey = pSupporter->rawEKey; pQuery->ekey = pSupporter->rawEKey;
pSupporter->meterIdx++; pSupporter->meterIdx++;
pQInfo->pMeterQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey;
// if the buffer is full or group by each table, we need to jump out of the loop // if the buffer is full or group by each table, we need to jump out of the loop
if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL) || if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL) ||
@ -770,7 +762,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
break; break;
} }
} else { // forward query range } else { // forward query range
pQuery->skey = pQuery->lastKey; pQuery->skey = pQuery->lastKey;
// all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter
@ -778,7 +770,7 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
assert(!Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); assert(!Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL));
continue; continue;
} else { } else {
pQInfo->pMeterQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey; pQInfo->pTableQuerySupporter->pMeterSidExtInfo[k]->key = pQuery->lastKey;
// buffer is full, wait for the next round to retrieve data from current meter // buffer is full, wait for the next round to retrieve data from current meter
assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)); assert(Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL));
break; break;
@ -808,20 +800,21 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
// todo refactor // todo refactor
if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
SSlidingWindowInfo* pSlidingWindowInfo = &pRuntimeEnv->swindowResInfo; SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
for (int32_t i = 0; i < pSlidingWindowInfo->size; ++i) { for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
SOutputRes *buf = &pSlidingWindowInfo->pResult[i]; SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status;
pSlidingWindowInfo->pStatus[i].closed = true; // enable return all results for group by normal columns pStatus->closed = true; // enable return all results for group by normal columns
SWindowResult *pResult = &pWindowResInfo->pResult[i];
for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) { for (int32_t j = 0; j < pQuery->numOfOutputCols; ++j) {
buf->numOfRows = MAX(buf->numOfRows, buf->resultInfo[j].numOfRes); pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
} }
} }
pQInfo->pMeterQuerySupporter->subgroupIdx = 0; pQInfo->pTableQuerySupporter->subgroupIdx = 0;
pQuery->pointsRead = 0; pQuery->pointsRead = 0;
copyFromGroupBuf(pQInfo, pSlidingWindowInfo->pResult); copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult);
} }
pQInfo->pointsRead += pQuery->pointsRead; pQInfo->pointsRead += pQuery->pointsRead;
@ -830,12 +823,12 @@ static void vnodeSTableSeqProcessor(SQInfo *pQInfo) {
dTrace( dTrace(
"QInfo %p vid:%d, numOfMeters:%d, index:%d, numOfGroups:%d, %d points returned, totalRead:%d totalReturn:%d," "QInfo %p vid:%d, numOfMeters:%d, index:%d, numOfGroups:%d, %d points returned, totalRead:%d totalReturn:%d,"
"next skey:%" PRId64 ", offset:%" PRId64, "next skey:%" PRId64 ", offset:%" PRId64,
pQInfo, vid, pSids->numOfSids, pSupporter->meterIdx, pSids->numOfSubSet, pQuery->pointsRead, pQInfo, vid, pSids->numOfSids, pSupporter->meterIdx, pSids->numOfSubSet, pQuery->pointsRead, pQInfo->pointsRead,
pQInfo->pointsRead, pQInfo->pointsReturned, pQuery->skey, pQuery->limit.offset); pQInfo->pointsReturned, pQuery->skey, pQuery->limit.offset);
} }
static void doOrderedScan(SQInfo *pQInfo) { static void doOrderedScan(SQInfo *pQInfo) {
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SQuery * pQuery = &pQInfo->query; SQuery * pQuery = &pQInfo->query;
if (QUERY_IS_ASC_QUERY(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) {
@ -855,17 +848,17 @@ static void doOrderedScan(SQInfo *pQInfo) {
} }
} }
static void setupMeterQueryInfoForSupplementQuery(SMeterQuerySupportObj *pSupporter) { static void setupMeterQueryInfoForSupplementQuery(STableQuerySupportObj *pSupporter) {
SQuery *pQuery = pSupporter->runtimeEnv.pQuery;
for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) { for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) {
SMeterQueryInfo *pMeterQueryInfo = pSupporter->pMeterDataInfo[i].pMeterQInfo; SMeterQueryInfo *pMeterQueryInfo = pSupporter->pMeterDataInfo[i].pMeterQInfo;
SQueryResultBuf* pResultBuf = pSupporter->runtimeEnv.pResultBuf; changeMeterQueryInfoForSuppleQuery(pQuery, pMeterQueryInfo, pSupporter->rawSKey, pSupporter->rawEKey);
changeMeterQueryInfoForSuppleQuery(pResultBuf, pMeterQueryInfo, pSupporter->rawSKey, pSupporter->rawEKey);
} }
} }
static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) { static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) {
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv; SQueryRuntimeEnv *pRuntimeEnv = &pSupporter->runtimeEnv;
SQuery * pQuery = &pQInfo->query; SQuery * pQuery = &pQInfo->query;
@ -876,7 +869,7 @@ static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) {
} }
SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv); SET_SUPPLEMENT_SCAN_FLAG(pRuntimeEnv);
disableFunctForSuppleScan(pRuntimeEnv, pQuery->order.order); disableFunctForSuppleScan(pSupporter, pQuery->order.order);
if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->pTSBuf != NULL) {
pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1; pRuntimeEnv->pTSBuf->cur.order = pRuntimeEnv->pTSBuf->cur.order ^ 1;
@ -886,7 +879,9 @@ static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) {
setupMeterQueryInfoForSupplementQuery(pSupporter); setupMeterQueryInfoForSupplementQuery(pSupporter);
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
doOrderedScan(pQInfo); doOrderedScan(pQInfo);
int64_t et = taosGetTimestampMs(); int64_t et = taosGetTimestampMs();
dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st); dTrace("QInfo:%p supplementary scan completed, elapsed time: %lldms", pQInfo, et - st);
@ -905,7 +900,8 @@ static void doMultiMeterSupplementaryScan(SQInfo *pQInfo) {
} }
static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) {
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv;
SQuery * pQuery = &pQInfo->query; SQuery * pQuery = &pQInfo->query;
if (pSupporter->subgroupIdx > 0) { if (pSupporter->subgroupIdx > 0) {
@ -913,14 +909,14 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) {
* if the subgroupIdx > 0, the query process must be completed yet, we only need to * if the subgroupIdx > 0, the query process must be completed yet, we only need to
* copy the data into output buffer * copy the data into output buffer
*/ */
if (pQuery->nAggTimeInterval > 0) { if (pQuery->intervalTime > 0) {
copyResToQueryResultBuf(pSupporter, pQuery); copyResToQueryResultBuf(pSupporter, pQuery);
#ifdef _DEBUG_VIEW #ifdef _DEBUG_VIEW
displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len);
#endif #endif
} else { } else {
copyFromGroupBuf(pQInfo, pSupporter->pResult); copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
} }
pQInfo->pointsRead += pQuery->pointsRead; pQInfo->pointsRead += pQuery->pointsRead;
@ -941,8 +937,8 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) {
return; return;
} }
dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, group:%d", pQInfo, pSupporter->rawSKey, pSupporter->rawEKey, dTrace("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, group:%d", pQInfo, pSupporter->rawSKey,
pQuery->order.order, pSupporter->pSidSet->numOfSubSet); pSupporter->rawEKey, pQuery->order.order, pSupporter->pSidSet->numOfSubSet);
dTrace("QInfo:%p main query scan start", pQInfo); dTrace("QInfo:%p main query scan start", pQInfo);
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
@ -951,12 +947,15 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) {
dTrace("QInfo:%p main scan completed, elapsed time: %lldms, supplementary scan start, order:%d", pQInfo, et - st, dTrace("QInfo:%p main scan completed, elapsed time: %lldms, supplementary scan start, order:%d", pQInfo, et - st,
pQuery->order.order ^ 1); pQuery->order.order ^ 1);
// failed to save all intermediate results into disk, abort further query processing if (pQuery->intervalTime > 0) {
if (doCloseAllOpenedResults(pSupporter) != TSDB_CODE_SUCCESS) { for (int32_t i = 0; i < pSupporter->numOfMeters; ++i) {
dError("QInfo:%p failed to save intermediate results, abort further query processing", pQInfo); SMeterQueryInfo *pMeterQueryInfo = pSupporter->pMeterDataInfo[i].pMeterQInfo;
return; closeAllTimeWindow(&pMeterQueryInfo->windowResInfo);
}
} else { // close results for group result
closeAllTimeWindow(&pRuntimeEnv->windowResInfo);
} }
doMultiMeterSupplementaryScan(pQInfo); doMultiMeterSupplementaryScan(pQInfo);
if (isQueryKilled(pQuery)) { if (isQueryKilled(pQuery)) {
@ -964,18 +963,18 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) {
return; return;
} }
if (pQuery->nAggTimeInterval > 0 || isSumAvgRateQuery(pQuery)) { if (pQuery->intervalTime > 0 || isSumAvgRateQuery(pQuery)) {
assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0); assert(pSupporter->subgroupIdx == 0 && pSupporter->numOfGroupResultPages == 0);
if (mergeMetersResultToOneGroups(pSupporter) == TSDB_CODE_SUCCESS) { if (mergeMetersResultToOneGroups(pSupporter) == TSDB_CODE_SUCCESS) {
copyResToQueryResultBuf(pSupporter, pQuery); copyResToQueryResultBuf(pSupporter, pQuery);
#ifdef _DEBUG_VIEW #ifdef _DEBUG_VIEW
displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len); displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->len);
#endif #endif
} }
} else { // not a interval query } else { // not a interval query
copyFromGroupBuf(pQInfo, pSupporter->pResult); copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
} }
// handle the limitation of output buffer // handle the limitation of output buffer
@ -992,7 +991,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) {
*/ */
static void vnodeSingleTableFixedOutputProcessor(SQInfo *pQInfo) { static void vnodeSingleTableFixedOutputProcessor(SQInfo *pQInfo) {
SQuery * pQuery = &pQInfo->query; SQuery * pQuery = &pQInfo->query;
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pMeterQuerySupporter->runtimeEnv; SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pTableQuerySupporter->runtimeEnv;
assert(pQuery->slot >= 0 && pQuery->pos >= 0); assert(pQuery->slot >= 0 && pQuery->pos >= 0);
@ -1023,7 +1022,7 @@ static void vnodeSingleTableMultiOutputProcessor(SQInfo *pQInfo) {
SQuery * pQuery = &pQInfo->query; SQuery * pQuery = &pQInfo->query;
SMeterObj *pMeterObj = pQInfo->pObj; SMeterObj *pMeterObj = pQInfo->pObj;
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pMeterQuerySupporter->runtimeEnv; SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pTableQuerySupporter->runtimeEnv;
// for ts_comp query, re-initialized is not allowed // for ts_comp query, re-initialized is not allowed
if (!isTSCompQuery(pQuery)) { if (!isTSCompQuery(pQuery)) {
@ -1054,8 +1053,9 @@ static void vnodeSingleTableMultiOutputProcessor(SQInfo *pQInfo) {
TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos);
assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)));
dTrace("QInfo:%p vid:%d sid:%d id:%s, skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, pQInfo, dTrace("QInfo:%p vid:%d sid:%d id:%s, skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64,
pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->limit.offset, pQuery->lastKey, pQuery->ekey); pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->limit.offset, pQuery->lastKey,
pQuery->ekey);
resetCtxOutputBuf(pRuntimeEnv); resetCtxOutputBuf(pRuntimeEnv);
} }
@ -1067,8 +1067,8 @@ static void vnodeSingleTableMultiOutputProcessor(SQInfo *pQInfo) {
TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos); TSKEY nextTimestamp = loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos);
assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK))); assert(nextTimestamp > 0 || ((nextTimestamp < 0) && Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)));
dTrace("QInfo:%p vid:%d sid:%d id:%s, query abort due to buffer limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, dTrace("QInfo:%p vid:%d sid:%d id:%s, query abort due to buffer limitation, next qrange:%" PRId64 "-%" PRId64,
pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->lastKey, pQuery->ekey); pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->lastKey, pQuery->ekey);
} }
dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode, dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned, totalRead:%d totalReturn:%d", pQInfo, pMeterObj->vnode,
@ -1080,58 +1080,39 @@ static void vnodeSingleTableMultiOutputProcessor(SQInfo *pQInfo) {
} }
} }
static void vnodeSingleMeterIntervalMainLooper(SMeterQuerySupportObj *pSupporter, SQueryRuntimeEnv *pRuntimeEnv) { static void vnodeSingleMeterIntervalMainLooper(STableQuerySupportObj *pSupporter, SQueryRuntimeEnv *pRuntimeEnv) {
SQuery *pQuery = pRuntimeEnv->pQuery; SQuery *pQuery = pRuntimeEnv->pQuery;
while (1) { while (1) {
assert((pQuery->skey <= pQuery->ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
(pQuery->skey >= pQuery->ekey && !QUERY_IS_ASC_QUERY(pQuery)));
initCtxOutputBuf(pRuntimeEnv); initCtxOutputBuf(pRuntimeEnv);
clearCompletedSlidingWindows(pRuntimeEnv);
vnodeScanAllData(pRuntimeEnv); vnodeScanAllData(pRuntimeEnv);
if (isQueryKilled(pQuery)) { if (isQueryKilled(pQuery)) {
return; return;
} }
assert(!Q_STATUS_EQUAL(pQuery->over, QUERY_NOT_COMPLETED)); assert(!Q_STATUS_EQUAL(pQuery->over, QUERY_NOT_COMPLETED));
// clear tag, used to decide if the whole interval query is completed or not
pQuery->over &= (~QUERY_COMPLETED);
doFinalizeResult(pRuntimeEnv); doFinalizeResult(pRuntimeEnv);
int64_t maxOutput = getNumOfResult(pRuntimeEnv);
// here we can ignore the records in case of no interpolation // here we can ignore the records in case of no interpolation
// todo handle offset, in case of top/bottom interval query
if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) && pQuery->limit.offset > 0 && if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) && pQuery->limit.offset > 0 &&
pQuery->interpoType == TSDB_INTERPO_NONE) { // maxOutput <= 0, means current query does not generate any results pQuery->interpoType == TSDB_INTERPO_NONE) {
// todo handle offset, in case of top/bottom interval query // maxOutput <= 0, means current query does not generate any results
if (maxOutput > 0) { int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo);
pQuery->limit.offset--;
} int32_t c = MIN(numOfClosed, pQuery->limit.offset);
} else { clearFirstNTimeWindow(pRuntimeEnv, c);
pQuery->pointsRead += maxOutput; pQuery->limit.offset -= c;
forwardCtxOutputBuf(pRuntimeEnv, maxOutput);
} }
if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK)) { if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) {
break; break;
} }
forwardIntervalQueryRange(pSupporter, pRuntimeEnv); // load the data block for the next retrieve
if (Q_STATUS_EQUAL(pQuery->over, QUERY_COMPLETED|QUERY_RESBUF_FULL)) { loadRequiredBlockIntoMem(pRuntimeEnv, &pRuntimeEnv->nextPos);
break; if (Q_STATUS_EQUAL(pQuery->over, QUERY_RESBUF_FULL)) {
}
/*
* the scan limitation mechanism is upon here,
* 1. since there is only one(k) record is generated in one scan operation
* 2. remain space is not sufficient for next query output, abort
*/
if ((pQuery->pointsRead % pQuery->pointsToRead == 0 && pQuery->pointsRead != 0) ||
((pQuery->pointsRead + maxOutput) > pQuery->pointsToRead)) {
setQueryStatus(pQuery, QUERY_RESBUF_FULL);
break; break;
} }
} }
@ -1142,7 +1123,7 @@ static void vnodeSingleTableIntervalProcessor(SQInfo *pQInfo) {
SQuery * pQuery = &(pQInfo->query); SQuery * pQuery = &(pQInfo->query);
SMeterObj *pMeterObj = pQInfo->pObj; SMeterObj *pMeterObj = pQInfo->pObj;
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv; SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv;
int32_t numOfInterpo = 0; int32_t numOfInterpo = 0;
@ -1151,6 +1132,14 @@ static void vnodeSingleTableIntervalProcessor(SQInfo *pQInfo) {
resetCtxOutputBuf(pRuntimeEnv); resetCtxOutputBuf(pRuntimeEnv);
vnodeSingleMeterIntervalMainLooper(pSupporter, pRuntimeEnv); vnodeSingleMeterIntervalMainLooper(pSupporter, pRuntimeEnv);
if (pQuery->intervalTime > 0) {
pSupporter->subgroupIdx = 0; // always start from 0
pQuery->pointsRead = 0;
copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
clearFirstNTimeWindow(pRuntimeEnv, pSupporter->subgroupIdx);
}
// the offset is handled at prepare stage if no interpolation involved // the offset is handled at prepare stage if no interpolation involved
if (pQuery->interpoType == TSDB_INTERPO_NONE) { if (pQuery->interpoType == TSDB_INTERPO_NONE) {
doRevisedResultsByLimit(pQInfo); doRevisedResultsByLimit(pQInfo);
@ -1177,11 +1166,13 @@ static void vnodeSingleTableIntervalProcessor(SQInfo *pQInfo) {
pQuery->pointsRead = 0; pQuery->pointsRead = 0;
} }
} }
if (isGroupbyNormalCol(pQuery->pGroupbyExpr) || (pQuery->slidingTime > 0 && pQuery->nAggTimeInterval > 0)) { // all data scanned, the group by normal column can return
pQInfo->pMeterQuerySupporter->subgroupIdx = 0; if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) {//todo refactor with merge interval time result
pSupporter->subgroupIdx = 0;
pQuery->pointsRead = 0; pQuery->pointsRead = 0;
copyFromGroupBuf(pQInfo, pRuntimeEnv->swindowResInfo.pResult); copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
clearFirstNTimeWindow(pRuntimeEnv, pSupporter->subgroupIdx);
} }
pQInfo->pointsRead += pQuery->pointsRead; pQInfo->pointsRead += pQuery->pointsRead;
@ -1195,7 +1186,7 @@ static void vnodeSingleTableIntervalProcessor(SQInfo *pQInfo) {
void vnodeSingleTableQuery(SSchedMsg *pMsg) { void vnodeSingleTableQuery(SSchedMsg *pMsg) {
SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; SQInfo *pQInfo = (SQInfo *)pMsg->ahandle;
if (pQInfo == NULL || pQInfo->pMeterQuerySupporter == NULL) { if (pQInfo == NULL || pQInfo->pTableQuerySupporter == NULL) {
dTrace("%p freed abort query", pQInfo); dTrace("%p freed abort query", pQInfo);
return; return;
} }
@ -1203,21 +1194,22 @@ void vnodeSingleTableQuery(SSchedMsg *pMsg) {
if (pQInfo->killed) { if (pQInfo->killed) {
dTrace("QInfo:%p it is already killed, abort", pQInfo); dTrace("QInfo:%p it is already killed, abort", pQInfo);
vnodeDecRefCount(pQInfo); vnodeDecRefCount(pQInfo);
return; return;
} }
assert(pQInfo->refCount >= 1); assert(pQInfo->refCount >= 1);
SQuery * pQuery = &pQInfo->query; SQuery * pQuery = &pQInfo->query;
SMeterObj *pMeterObj = pQInfo->pObj; SMeterObj * pMeterObj = pQInfo->pObj;
STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
SQueryRuntimeEnv * pRuntimeEnv = &pSupporter->runtimeEnv;
assert(pRuntimeEnv->pMeterObj == pMeterObj);
dTrace("vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, QInfo:%p", pMeterObj->vnode, pMeterObj->sid, dTrace("vid:%d sid:%d id:%s, query thread is created, numOfQueries:%d, QInfo:%p", pMeterObj->vnode, pMeterObj->sid,
pMeterObj->meterId, pMeterObj->numOfQueries, pQInfo); pMeterObj->meterId, pMeterObj->numOfQueries, pQInfo);
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->pMeterQuerySupporter->runtimeEnv;
assert(pRuntimeEnv->pMeterObj == pMeterObj);
if (vnodeHasRemainResults(pQInfo)) { if (vnodeHasRemainResults(pQInfo)) {
/* /*
* There are remain results that are not returned due to result interpolation * There are remain results that are not returned due to result interpolation
@ -1249,12 +1241,18 @@ void vnodeSingleTableQuery(SSchedMsg *pMsg) {
// here we have scan all qualified data in both data file and cache // here we have scan all qualified data in both data file and cache
if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) { if (Q_STATUS_EQUAL(pQuery->over, QUERY_NO_DATA_TO_CHECK | QUERY_COMPLETED)) {
// continue to get push data from the group result // continue to get push data from the group result
if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { if (isGroupbyNormalCol(pQuery->pGroupbyExpr) ||
(pQuery->intervalTime > 0 && pQInfo->pointsReturned < pQuery->limit.limit)) {
//todo limit the output for interval query?
pQuery->pointsRead = 0; pQuery->pointsRead = 0;
if (pQInfo->pMeterQuerySupporter->subgroupIdx > 0) { pSupporter->subgroupIdx = 0; // always start from 0
copyFromGroupBuf(pQInfo, pQInfo->pMeterQuerySupporter->pResult);
if (pRuntimeEnv->windowResInfo.size > 0) {
copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult);
pQInfo->pointsRead += pQuery->pointsRead; pQInfo->pointsRead += pQuery->pointsRead;
clearFirstNTimeWindow(pRuntimeEnv, pSupporter->subgroupIdx);
if (pQuery->pointsRead > 0) { if (pQuery->pointsRead > 0) {
dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d totalReturn:%d", dTrace("QInfo:%p vid:%d sid:%d id:%s, %d points returned %d from group results, totalRead:%d totalReturn:%d",
pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead,
@ -1262,19 +1260,19 @@ void vnodeSingleTableQuery(SSchedMsg *pMsg) {
sem_post(&pQInfo->dataReady); sem_post(&pQInfo->dataReady);
vnodeDecRefCount(pQInfo); vnodeDecRefCount(pQInfo);
return; return;
} }
} }
} }
pQInfo->over = 1; pQInfo->over = 1;
dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, pMeterObj->sid,
pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQInfo->pointsRead); pMeterObj->meterId, pQInfo->pointsRead);
vnodePrintQueryStatistics(pQInfo->pMeterQuerySupporter); vnodePrintQueryStatistics(pSupporter);
sem_post(&pQInfo->dataReady); sem_post(&pQInfo->dataReady);
vnodeDecRefCount(pQInfo); vnodeDecRefCount(pQInfo);
return; return;
} }
@ -1286,7 +1284,7 @@ void vnodeSingleTableQuery(SSchedMsg *pMsg) {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
// group by normal column, sliding window query, interval query are handled by interval query processor // group by normal column, sliding window query, interval query are handled by interval query processor
if (pQuery->nAggTimeInterval != 0 || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation) if (pQuery->intervalTime != 0 || isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // interval (down sampling operation)
assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead); assert(pQuery->checkBufferInLoop == 0 && pQuery->pointsOffset == pQuery->pointsToRead);
vnodeSingleTableIntervalProcessor(pQInfo); vnodeSingleTableIntervalProcessor(pQInfo);
} else { } else {
@ -1307,8 +1305,8 @@ void vnodeSingleTableQuery(SSchedMsg *pMsg) {
dTrace("QInfo:%p query is killed", pQInfo); dTrace("QInfo:%p query is killed", pQInfo);
pQInfo->over = 1; pQInfo->over = 1;
} else { } else {
dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo,
pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead);
} }
sem_post(&pQInfo->dataReady); sem_post(&pQInfo->dataReady);
@ -1318,7 +1316,7 @@ void vnodeSingleTableQuery(SSchedMsg *pMsg) {
void vnodeMultiMeterQuery(SSchedMsg *pMsg) { void vnodeMultiMeterQuery(SSchedMsg *pMsg) {
SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; SQInfo *pQInfo = (SQInfo *)pMsg->ahandle;
if (pQInfo == NULL || pQInfo->pMeterQuerySupporter == NULL) { if (pQInfo == NULL || pQInfo->pTableQuerySupporter == NULL) {
return; return;
} }
@ -1334,12 +1332,12 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) {
pQuery->pointsRead = 0; pQuery->pointsRead = 0;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
if (pQuery->nAggTimeInterval > 0 || if (pQuery->intervalTime > 0 ||
(isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) { (isFixedOutputQuery(pQuery) && (!isPointInterpoQuery(pQuery)) && !isGroupbyNormalCol(pQuery->pGroupbyExpr))) {
assert(pQuery->checkBufferInLoop == 0); assert(pQuery->checkBufferInLoop == 0);
vnodeMultiMeterQueryProcessor(pQInfo); vnodeMultiMeterQueryProcessor(pQInfo);
} else { } else {
assert((pQuery->checkBufferInLoop == 1 && pQuery->nAggTimeInterval == 0) || isPointInterpoQuery(pQuery) || assert((pQuery->checkBufferInLoop == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) ||
isGroupbyNormalCol(pQuery->pGroupbyExpr)); isGroupbyNormalCol(pQuery->pGroupbyExpr));
vnodeSTableSeqProcessor(pQInfo); vnodeSTableSeqProcessor(pQInfo);
@ -1349,10 +1347,10 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) {
pQInfo->useconds += (taosGetTimestampUs() - st); pQInfo->useconds += (taosGetTimestampUs() - st);
pQInfo->over = isQueryKilled(pQuery) ? 1 : 0; pQInfo->over = isQueryKilled(pQuery) ? 1 : 0;
taosInterpoSetStartInfo(&pQInfo->pMeterQuerySupporter->runtimeEnv.interpoInfo, pQuery->pointsRead, taosInterpoSetStartInfo(&pQInfo->pTableQuerySupporter->runtimeEnv.interpoInfo, pQuery->pointsRead,
pQInfo->query.interpoType); pQInfo->query.interpoType);
SMeterQuerySupportObj *pSupporter = pQInfo->pMeterQuerySupporter; STableQuerySupportObj *pSupporter = pQInfo->pTableQuerySupporter;
if (pQuery->pointsRead == 0) { if (pQuery->pointsRead == 0) {
pQInfo->over = 1; pQInfo->over = 1;

26
src/system/detail/src/vnodeRead.c
View File

@ -266,7 +266,7 @@ static SQInfo *vnodeAllocateQInfoEx(SQueryMeterMsg *pQueryMsg, SSqlGroupbyExpr *
} }
pQuery->pGroupbyExpr = pGroupbyExpr; pQuery->pGroupbyExpr = pGroupbyExpr;
pQuery->nAggTimeInterval = pQueryMsg->nAggTimeInterval; pQuery->intervalTime = pQueryMsg->intervalTime;
pQuery->slidingTime = pQueryMsg->slidingTime; pQuery->slidingTime = pQueryMsg->slidingTime;
pQuery->interpoType = pQueryMsg->interpoType; pQuery->interpoType = pQueryMsg->interpoType;
pQuery->intervalTimeUnit = pQueryMsg->intervalTimeUnit; pQuery->intervalTimeUnit = pQueryMsg->intervalTimeUnit;
@ -648,7 +648,7 @@ void *vnodeQueryOnSingleTable(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE
goto _error; goto _error;
} }
SMeterQuerySupportObj *pSupporter = (SMeterQuerySupportObj *)calloc(1, sizeof(SMeterQuerySupportObj)); STableQuerySupportObj *pSupporter = (STableQuerySupportObj *)calloc(1, sizeof(STableQuerySupportObj));
pSupporter->numOfMeters = 1; pSupporter->numOfMeters = 1;
pSupporter->pMetersHashTable = taosInitHashTable(pSupporter->numOfMeters, taosIntHash_32, false); pSupporter->pMetersHashTable = taosInitHashTable(pSupporter->numOfMeters, taosIntHash_32, false);
@ -659,7 +659,7 @@ void *vnodeQueryOnSingleTable(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE
pSupporter->subgroupIdx = -1; pSupporter->subgroupIdx = -1;
pSupporter->pMeterSidExtInfo = NULL; pSupporter->pMeterSidExtInfo = NULL;
pQInfo->pMeterQuerySupporter = pSupporter; pQInfo->pTableQuerySupporter = pSupporter;
STSBuf *pTSBuf = NULL; STSBuf *pTSBuf = NULL;
if (pQueryMsg->tsLen > 0) { if (pQueryMsg->tsLen > 0) {
@ -670,7 +670,7 @@ void *vnodeQueryOnSingleTable(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE
tsBufNextPos(pTSBuf); tsBufNextPos(pTSBuf);
} }
if (((*code) = vnodeQuerySingleMeterPrepare(pQInfo, pQInfo->pObj, pSupporter, pTSBuf)) != TSDB_CODE_SUCCESS) { if (((*code) = vnodeQueryTablePrepare(pQInfo, pQInfo->pObj, pSupporter, pTSBuf)) != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
@ -739,7 +739,7 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE
SSchedMsg schedMsg = {0}; SSchedMsg schedMsg = {0};
SMeterQuerySupportObj *pSupporter = (SMeterQuerySupportObj *)calloc(1, sizeof(SMeterQuerySupportObj)); STableQuerySupportObj *pSupporter = (STableQuerySupportObj *)calloc(1, sizeof(STableQuerySupportObj));
pSupporter->numOfMeters = pQueryMsg->numOfSids; pSupporter->numOfMeters = pQueryMsg->numOfSids;
pSupporter->pMetersHashTable = taosInitHashTable(pSupporter->numOfMeters, taosIntHash_32, false); pSupporter->pMetersHashTable = taosInitHashTable(pSupporter->numOfMeters, taosIntHash_32, false);
@ -784,7 +784,7 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE
(SSchema *)pQueryMsg->pTagSchema, pQueryMsg->numOfTagsCols, NULL, 0); (SSchema *)pQueryMsg->pTagSchema, pQueryMsg->numOfTagsCols, NULL, 0);
} }
pQInfo->pMeterQuerySupporter = pSupporter; pQInfo->pTableQuerySupporter = pSupporter;
STSBuf *pTSBuf = NULL; STSBuf *pTSBuf = NULL;
if (pQueryMsg->tsLen > 0) { if (pQueryMsg->tsLen > 0) {
@ -794,7 +794,7 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE
tsBufResetPos(pTSBuf); tsBufResetPos(pTSBuf);
} }
if (((*code) = vnodeMultiMeterQueryPrepare(pQInfo, pQuery, pTSBuf)) != TSDB_CODE_SUCCESS) { if (((*code) = vnodeSTableQueryPrepare(pQInfo, pQuery, pTSBuf)) != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
@ -898,8 +898,8 @@ int vnodeSaveQueryResult(void *handle, char *data, int32_t *size) {
SSchedMsg schedMsg = {0}; SSchedMsg schedMsg = {0};
if (pQInfo->pMeterQuerySupporter != NULL) { if (pQInfo->pTableQuerySupporter != NULL) {
if (pQInfo->pMeterQuerySupporter->pSidSet == NULL) { if (pQInfo->pTableQuerySupporter->pSidSet == NULL) {
schedMsg.fp = vnodeSingleTableQuery; schedMsg.fp = vnodeSingleTableQuery;
} else { // group by tag } else { // group by tag
schedMsg.fp = vnodeMultiMeterQuery; schedMsg.fp = vnodeMultiMeterQuery;
@ -920,8 +920,8 @@ int vnodeSaveQueryResult(void *handle, char *data, int32_t *size) {
} }
static int32_t validateQueryMeterMsg(SQueryMeterMsg *pQueryMsg) { static int32_t validateQueryMeterMsg(SQueryMeterMsg *pQueryMsg) {
if (pQueryMsg->nAggTimeInterval < 0) { if (pQueryMsg->intervalTime < 0) {
dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryMsg, pQueryMsg->nAggTimeInterval); dError("qmsg:%p illegal value of aggTimeInterval %" PRId64 "", pQueryMsg, pQueryMsg->intervalTime);
return -1; return -1;
} }
@ -975,7 +975,7 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) {
pQueryMsg->queryType = htons(pQueryMsg->queryType); pQueryMsg->queryType = htons(pQueryMsg->queryType);
pQueryMsg->nAggTimeInterval = htobe64(pQueryMsg->nAggTimeInterval); pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime);
pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime); pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime);
pQueryMsg->numOfTagsCols = htons(pQueryMsg->numOfTagsCols); pQueryMsg->numOfTagsCols = htons(pQueryMsg->numOfTagsCols);
@ -1146,7 +1146,7 @@ int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQueryMsg) {
"offset:%" PRId64, "offset:%" PRId64,
pQueryMsg, pQueryMsg->numOfSids, pQueryMsg->skey, pQueryMsg->ekey, pQueryMsg->numOfGroupCols, pQueryMsg, pQueryMsg->numOfSids, pQueryMsg->skey, pQueryMsg->ekey, pQueryMsg->numOfGroupCols,
pQueryMsg->numOfTagsCols, pQueryMsg->order, pQueryMsg->orderType, pQueryMsg->orderByIdx, pQueryMsg->numOfTagsCols, pQueryMsg->order, pQueryMsg->orderType, pQueryMsg->orderByIdx,
pQueryMsg->numOfOutputCols, pQueryMsg->numOfCols, pQueryMsg->nAggTimeInterval, pQueryMsg->interpoType, pQueryMsg->numOfOutputCols, pQueryMsg->numOfCols, pQueryMsg->intervalTime, pQueryMsg->interpoType,
pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset); pQueryMsg->tsLen, pQueryMsg->limit, pQueryMsg->offset);
return 0; return 0;

14
src/system/detail/src/vnodeShell.c
View File

@ -446,8 +446,8 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) {
// buffer size for progress information, including meter count, // buffer size for progress information, including meter count,
// and for each meter, including 'uid' and 'TSKEY'. // and for each meter, including 'uid' and 'TSKEY'.
int progressSize = 0; int progressSize = 0;
if (pQInfo->pMeterQuerySupporter != NULL) if (pQInfo->pTableQuerySupporter != NULL)
progressSize = pQInfo->pMeterQuerySupporter->numOfMeters * (sizeof(int64_t) + sizeof(TSKEY)) + sizeof(int32_t); progressSize = pQInfo->pTableQuerySupporter->numOfMeters * (sizeof(int64_t) + sizeof(TSKEY)) + sizeof(int32_t);
else if (pQInfo->pObj != NULL) else if (pQInfo->pObj != NULL)
progressSize = sizeof(int64_t) + sizeof(TSKEY) + sizeof(int32_t); progressSize = sizeof(int64_t) + sizeof(TSKEY) + sizeof(int32_t);
@ -486,13 +486,13 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) {
// write the progress information of each meter to response // write the progress information of each meter to response
// this is required by subscriptions // this is required by subscriptions
if (pQInfo != NULL ) { if (pQInfo != NULL ) {
if (pQInfo->pMeterQuerySupporter != NULL && pQInfo->pMeterQuerySupporter->pMeterSidExtInfo != NULL) { if (pQInfo->pTableQuerySupporter != NULL && pQInfo->pTableQuerySupporter->pMeterSidExtInfo != NULL) {
*((int32_t *)pMsg) = htonl(pQInfo->pMeterQuerySupporter->numOfMeters); *((int32_t *)pMsg) = htonl(pQInfo->pTableQuerySupporter->numOfMeters);
pMsg += sizeof(int32_t); pMsg += sizeof(int32_t);
for (int32_t i = 0; i < pQInfo->pMeterQuerySupporter->numOfMeters; i++) { for (int32_t i = 0; i < pQInfo->pTableQuerySupporter->numOfMeters; i++) {
*((int64_t *)pMsg) = htobe64(pQInfo->pMeterQuerySupporter->pMeterSidExtInfo[i]->uid); *((int64_t *)pMsg) = htobe64(pQInfo->pTableQuerySupporter->pMeterSidExtInfo[i]->uid);
pMsg += sizeof(int64_t); pMsg += sizeof(int64_t);
*((TSKEY *)pMsg) = htobe64(pQInfo->pMeterQuerySupporter->pMeterSidExtInfo[i]->key); *((TSKEY *)pMsg) = htobe64(pQInfo->pTableQuerySupporter->pMeterSidExtInfo[i]->key);
pMsg += sizeof(TSKEY); pMsg += sizeof(TSKEY);
} }
} else if (pQInfo->pObj != NULL) { } else if (pQInfo->pObj != NULL) {

28
src/system/detail/src/vnodeUtil.c
View File

@ -372,18 +372,22 @@ void vnodeUpdateFilterColumnIndex(SQuery* pQuery) {
} }
// set the column index in buffer for arithmetic operation // set the column index in buffer for arithmetic operation
if (pQuery->pSelectExpr != NULL) { if (pQuery->pSelectExpr == NULL) {
for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) { return;
SSqlBinaryExprInfo* pBinExprInfo = &pQuery->pSelectExpr[i].pBinExprInfo; }
if (pBinExprInfo->pBinExpr != NULL) {
for (int16_t j = 0; j < pBinExprInfo->numOfCols; ++j) { for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) {
for (int32_t k = 0; k < pQuery->numOfCols; ++k) { SSqlBinaryExprInfo* pBinExprInfo = &pQuery->pSelectExpr[i].pBinExprInfo;
if (pBinExprInfo->pReqColumns[j].colId == pQuery->colList[k].data.colId) { if (pBinExprInfo->pBinExpr == NULL) {
pBinExprInfo->pReqColumns[j].colIdxInBuf = pQuery->colList[k].colIdxInBuf; continue;
assert(pQuery->colList[k].colIdxInBuf == k); }
break;
} for (int16_t j = 0; j < pBinExprInfo->numOfCols; ++j) {
} for (int32_t k = 0; k < pQuery->numOfCols; ++k) {
if (pBinExprInfo->pReqColumns[j].colId == pQuery->colList[k].data.colId) {
pBinExprInfo->pReqColumns[j].colIdxInBuf = pQuery->colList[k].colIdxInBuf;
assert(pQuery->colList[k].colIdxInBuf == k);
break;
} }
} }
} }

30
src/util/src/hash.c
View File

@ -340,10 +340,6 @@ static void doAddToHashTable(HashObj *pObj, SHashNode *pNode) {
pEntry->num++; pEntry->num++;
pObj->size++; pObj->size++;
// char key[512] = {0};
// memcpy(key, pNode->key, MIN(512, pNode->keyLen));
// pTrace("key:%s %p add to hash table", key, pNode);
} }
int32_t taosNumElemsInHashTable(HashObj *pObj) { int32_t taosNumElemsInHashTable(HashObj *pObj) {
@ -525,29 +521,3 @@ int32_t taosGetHashMaxOverflowLength(HashObj* pObj) {
return num; return num;
} }
int32_t taosCheckHashTable(HashObj *pObj) {
for(int32_t i = 0; i < pObj->capacity; ++i) {
SHashEntry *pEntry = pObj->hashList[i];
SHashNode* pNode = pEntry->next;
if (pNode != NULL) {
assert(pEntry == pNode->prev1);
int32_t num = 1;
SHashNode* pNext = pNode->next;
while(pNext) {
assert(pNext->prev == pNode);
pNode = pNext;
pNext = pNext->next;
num ++;
}
assert(num == pEntry->num);
}
}
return 0;
}

8
src/util/src/textbuffer.c
View File

@ -468,8 +468,8 @@ int32_t compare_a(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1,
return ret; return ret;
} }
} else { } else {
SSchema *pSchema = &pDescriptor->pColumnModel->pFields[colIdx]; SSchemaEx *pSchema = &pDescriptor->pColumnModel->pFields[colIdx];
int32_t ret = columnValueAscendingComparator(f1, f2, pSchema->type, pSchema->bytes); int32_t ret = columnValueAscendingComparator(f1, f2, pSchema->field.type, pSchema->field.bytes);
if (ret == 0) { if (ret == 0) {
continue; continue;
} else { } else {
@ -500,8 +500,8 @@ int32_t compare_d(tOrderDescriptor *pDescriptor, int32_t numOfRows1, int32_t s1,
return ret; return ret;
} }
} else { } else {
SSchema *pSchema = &pDescriptor->pColumnModel->pFields[colIdx]; SSchemaEx *pSchema = &pDescriptor->pColumnModel->pFields[colIdx];
int32_t ret = columnValueAscendingComparator(f1, f2, pSchema->type, pSchema->bytes); int32_t ret = columnValueAscendingComparator(f1, f2, pSchema->field.type, pSchema->field.bytes);
if (ret == 0) { if (ret == 0) {
continue; continue;
} else { } else {

132
src/util/src/tinterpolation.c
View File

@ -13,9 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <assert.h>
#include <stdint.h>
#include "os.h" #include "os.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "textbuffer.h" #include "textbuffer.h"
@ -47,7 +44,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t timeRange, char
char** tzname = _tzname; char** tzname = _tzname;
#endif #endif
int64_t t = (precision == TSDB_TIME_PRECISION_MILLI)?MILLISECOND_PER_SECOND:MILLISECOND_PER_SECOND*1000L; int64_t t = (precision == TSDB_TIME_PRECISION_MILLI) ? MILLISECOND_PER_SECOND : MILLISECOND_PER_SECOND * 1000L;
int64_t revStartime = (startTime / timeRange) * timeRange + timezone * t; int64_t revStartime = (startTime / timeRange) * timeRange + timezone * t;
int64_t revEndtime = revStartime + timeRange - 1; int64_t revEndtime = revStartime + timeRange - 1;
@ -78,14 +75,14 @@ void taosInitInterpoInfo(SInterpolationInfo* pInterpoInfo, int32_t order, int64_
} }
// the SInterpolationInfo itself will not be released // the SInterpolationInfo itself will not be released
void taosDestoryInterpoInfo(SInterpolationInfo *pInterpoInfo) { void taosDestoryInterpoInfo(SInterpolationInfo* pInterpoInfo) {
if (pInterpoInfo == NULL) { if (pInterpoInfo == NULL) {
return; return;
} }
tfree(pInterpoInfo->prevValues); tfree(pInterpoInfo->prevValues);
tfree(pInterpoInfo->nextValues); tfree(pInterpoInfo->nextValues);
tfree(pInterpoInfo->pTags); tfree(pInterpoInfo->pTags);
} }
@ -94,7 +91,7 @@ void taosInterpoSetStartInfo(SInterpolationInfo* pInterpoInfo, int32_t numOfRawD
return; return;
} }
pInterpoInfo->rowIdx = 0;//INTERPOL_IS_ASC_INTERPOL(pInterpoInfo) ? 0 : numOfRawDataInRows - 1; pInterpoInfo->rowIdx = 0; // INTERPOL_IS_ASC_INTERPOL(pInterpoInfo) ? 0 : numOfRawDataInRows - 1;
pInterpoInfo->numOfRawDataInRows = numOfRawDataInRows; pInterpoInfo->numOfRawDataInRows = numOfRawDataInRows;
} }
@ -118,14 +115,9 @@ int32_t taosGetNumOfResWithoutLimit(SInterpolationInfo* pInterpoInfo, int64_t* p
if (numOfAvailRawData > 0) { if (numOfAvailRawData > 0) {
int32_t finalNumOfResult = 0; int32_t finalNumOfResult = 0;
// if (pInterpoInfo->order == TSQL_SO_ASC) { // get last timestamp, calculate the result size
// get last timestamp, calculate the result size int64_t lastKey = pPrimaryKeyArray[pInterpoInfo->numOfRawDataInRows - 1];
int64_t lastKey = pPrimaryKeyArray[pInterpoInfo->numOfRawDataInRows - 1]; finalNumOfResult = (int32_t)(labs(lastKey - pInterpoInfo->startTimestamp) / nInterval) + 1;
finalNumOfResult = (int32_t)(labs(lastKey - pInterpoInfo->startTimestamp) / nInterval) + 1;
// } else { // todo error less than one!!!
// TSKEY lastKey = pPrimaryKeyArray[0];
// finalNumOfResult = (int32_t)((pInterpoInfo->startTimestamp - lastKey) / nInterval) + 1;
// }
assert(finalNumOfResult >= numOfAvailRawData); assert(finalNumOfResult >= numOfAvailRawData);
return finalNumOfResult; return finalNumOfResult;
@ -140,7 +132,9 @@ int32_t taosGetNumOfResWithoutLimit(SInterpolationInfo* pInterpoInfo, int64_t* p
} }
} }
bool taosHasRemainsDataForInterpolation(SInterpolationInfo* pInterpoInfo) { return taosNumOfRemainPoints(pInterpoInfo) > 0; } bool taosHasRemainsDataForInterpolation(SInterpolationInfo* pInterpoInfo) {
return taosNumOfRemainPoints(pInterpoInfo) > 0;
}
int32_t taosNumOfRemainPoints(SInterpolationInfo* pInterpoInfo) { int32_t taosNumOfRemainPoints(SInterpolationInfo* pInterpoInfo) {
if (pInterpoInfo->rowIdx == -1 || pInterpoInfo->numOfRawDataInRows == 0) { if (pInterpoInfo->rowIdx == -1 || pInterpoInfo->numOfRawDataInRows == 0) {
@ -197,28 +191,22 @@ int taosDoLinearInterpolation(int32_t type, SPoint* point1, SPoint* point2, SPoi
return 0; return 0;
} }
static char* getPos(char* data, int32_t bytes, int32_t order, int32_t capacity, int32_t index) { static char* getPos(char* data, int32_t bytes, int32_t index) { return data + index * bytes; }
// if (order == TSQL_SO_ASC) {
return data + index * bytes;
// } else {
// return data + (capacity - index - 1) * bytes;
// }
}
static void setTagsValueInInterpolation(tFilePage** data, char** pTags, SColumnModel* pModel, int32_t order, int32_t start, static void setTagsValueInInterpolation(tFilePage** data, char** pTags, SColumnModel* pModel, int32_t order,
int32_t capacity, int32_t num) { int32_t start, int32_t capacity, int32_t num) {
for (int32_t j = 0, i = start; i < pModel->numOfCols; ++i, ++j) { for (int32_t j = 0, i = start; i < pModel->numOfCols; ++i, ++j) {
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, order, capacity, num); char* val1 = getPos(data[i]->data, pSchema->bytes, num);
assignVal(val1, pTags[j], pSchema->bytes, pSchema->type); assignVal(val1, pTags[j], pSchema->bytes, pSchema->type);
} }
} }
static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interpoType, tFilePage** data, static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interpoType, tFilePage** data,
SColumnModel* pModel, int32_t* num, char** srcData, int64_t nInterval, int64_t* defaultVal, SColumnModel* pModel, int32_t* num, char** srcData, int64_t nInterval,
int64_t currentTimestamp, int32_t capacity, int32_t numOfTags, char** pTags, int64_t* defaultVal, int64_t currentTimestamp, int32_t capacity, int32_t numOfTags,
bool outOfBound) { char** pTags, bool outOfBound) {
char** prevValues = &pInterpoInfo->prevValues; char** prevValues = &pInterpoInfo->prevValues;
char** nextValues = &pInterpoInfo->nextValues; char** nextValues = &pInterpoInfo->nextValues;
@ -226,7 +214,7 @@ static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interp
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pInterpoInfo->order); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pInterpoInfo->order);
char* val = getPos(data[0]->data, TSDB_KEYSIZE, pInterpoInfo->order, capacity, *num); char* val = getPos(data[0]->data, TSDB_KEYSIZE, *num);
*(TSKEY*)val = pInterpoInfo->startTimestamp; *(TSKEY*)val = pInterpoInfo->startTimestamp;
int32_t numOfValCols = pModel->numOfCols - numOfTags; int32_t numOfValCols = pModel->numOfCols - numOfTags;
@ -237,9 +225,9 @@ static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interp
if (pInterpolationData != NULL) { if (pInterpolationData != NULL) {
for (int32_t i = 1; i < numOfValCols; ++i) { for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
int16_t offset = getColumnModelOffset(pModel, i); int16_t offset = getColumnModelOffset(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, pInterpoInfo->order, capacity, *num); char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
if (isNull(pInterpolationData + offset, pSchema->type)) { if (isNull(pInterpolationData + offset, pSchema->type)) {
setNull(val1, pSchema->type, pSchema->bytes); setNull(val1, pSchema->type, pSchema->bytes);
@ -250,8 +238,8 @@ static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interp
} else { /* no prev value yet, set the value for null */ } else { /* no prev value yet, set the value for null */
for (int32_t i = 1; i < numOfValCols; ++i) { for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, pInterpoInfo->order, capacity, *num); char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
setNull(val1, pSchema->type, pSchema->bytes); setNull(val1, pSchema->type, pSchema->bytes);
} }
} }
@ -262,10 +250,10 @@ static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interp
if (*prevValues != NULL && !outOfBound) { if (*prevValues != NULL && !outOfBound) {
for (int32_t i = 1; i < numOfValCols; ++i) { for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
int16_t offset = getColumnModelOffset(pModel, i); int16_t offset = getColumnModelOffset(pModel, i);
int16_t type = pSchema->type; int16_t type = pSchema->type;
char* val1 = getPos(data[i]->data, pSchema->bytes, pInterpoInfo->order, capacity, *num); char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
setNull(val1, type, pSchema->bytes); setNull(val1, type, pSchema->bytes);
@ -283,8 +271,8 @@ static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interp
} else { } else {
for (int32_t i = 1; i < numOfValCols; ++i) { for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, pInterpoInfo->order, capacity, *num); char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
setNull(val1, pSchema->type, pSchema->bytes); setNull(val1, pSchema->type, pSchema->bytes);
} }
@ -293,8 +281,8 @@ static void doInterpoResultImpl(SInterpolationInfo* pInterpoInfo, int16_t interp
} else { /* default value interpolation */ } else { /* default value interpolation */
for (int32_t i = 1; i < numOfValCols; ++i) { for (int32_t i = 1; i < numOfValCols; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, pInterpoInfo->order, capacity, *num); char* val1 = getPos(data[i]->data, pSchema->bytes, *num);
assignVal(val1, (char*)&defaultVal[i], pSchema->bytes, pSchema->type); assignVal(val1, (char*)&defaultVal[i], pSchema->bytes, pSchema->type);
} }
@ -344,9 +332,9 @@ int32_t taosDoInterpoResult(SInterpolationInfo* pInterpoInfo, int16_t interpoTyp
if (*nextValues == NULL) { if (*nextValues == NULL) {
*nextValues = calloc(1, pModel->rowSize); *nextValues = calloc(1, pModel->rowSize);
for (int i = 1; i < pModel->numOfCols; i++) { for (int i = 1; i < pModel->numOfCols; i++) {
int16_t offset = getColumnModelOffset(pModel, i); int16_t offset = getColumnModelOffset(pModel, i);
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
setNull(*nextValues + offset, pSchema->type, pSchema->bytes); setNull(*nextValues + offset, pSchema->type, pSchema->bytes);
} }
} }
@ -354,33 +342,36 @@ int32_t taosDoInterpoResult(SInterpolationInfo* pInterpoInfo, int16_t interpoTyp
int32_t offset = pInterpoInfo->rowIdx; int32_t offset = pInterpoInfo->rowIdx;
for (int32_t tlen = 0, i = 0; i < pModel->numOfCols - numOfTags; ++i) { for (int32_t tlen = 0, i = 0; i < pModel->numOfCols - numOfTags; ++i) {
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
memcpy(*nextValues + tlen, srcData[i] + offset * pSchema->bytes, pSchema->bytes); memcpy(*nextValues + tlen, srcData[i] + offset * pSchema->bytes, pSchema->bytes);
tlen += pSchema->bytes; tlen += pSchema->bytes;
} }
} }
while (((pInterpoInfo->startTimestamp < currentTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) || if (((pInterpoInfo->startTimestamp < currentTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
(pInterpoInfo->startTimestamp > currentTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) && (pInterpoInfo->startTimestamp > currentTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) &&
num < outputRows) { num < outputRows) {
doInterpoResultImpl(pInterpoInfo, interpoType, data, pModel, &num, srcData, nInterval, defaultVal, while (((pInterpoInfo->startTimestamp < currentTimestamp && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
currentTimestamp, bufSize, numOfTags, pTags, false); (pInterpoInfo->startTimestamp > currentTimestamp && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) &&
} num < outputRows) {
doInterpoResultImpl(pInterpoInfo, interpoType, data, pModel, &num, srcData, nInterval, defaultVal,
currentTimestamp, bufSize, numOfTags, pTags, false);
}
/* output buffer is full, abort */ /* output buffer is full, abort */
if ((num == outputRows && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) || if ((num == outputRows && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
(num < 0 && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) { (num < 0 && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo))) {
pInterpoInfo->numOfTotalInterpo += pInterpoInfo->numOfCurrentInterpo; pInterpoInfo->numOfTotalInterpo += pInterpoInfo->numOfCurrentInterpo;
return outputRows; return outputRows;
} }
} else {
if (pInterpoInfo->startTimestamp == currentTimestamp) { // if (pInterpoInfo->startTimestamp == currentTimestamp) {
if (*prevValues == NULL) { if (*prevValues == NULL) {
*prevValues = calloc(1, pModel->rowSize); *prevValues = calloc(1, pModel->rowSize);
for (int i = 1; i < pModel->numOfCols; i++) { for (int i = 1; i < pModel->numOfCols; i++) {
int16_t offset = getColumnModelOffset(pModel, i); int16_t offset = getColumnModelOffset(pModel, i);
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
setNull(*prevValues + offset, pSchema->type, pSchema->bytes); setNull(*prevValues + offset, pSchema->type, pSchema->bytes);
} }
} }
@ -388,17 +379,16 @@ int32_t taosDoInterpoResult(SInterpolationInfo* pInterpoInfo, int16_t interpoTyp
// assign rows to dst buffer // assign rows to dst buffer
int32_t i = 0; int32_t i = 0;
for (int32_t tlen = 0; i < pModel->numOfCols - numOfTags; ++i) { for (int32_t tlen = 0; i < pModel->numOfCols - numOfTags; ++i) {
int16_t offset = getColumnModelOffset(pModel, i); int16_t offset = getColumnModelOffset(pModel, i);
SSchema* pSchema = getColumnModelSchema(pModel, i); SSchema* pSchema = getColumnModelSchema(pModel, i);
char* val1 = getPos(data[i]->data, pSchema->bytes, pInterpoInfo->order, bufSize, num); char* val1 = getPos(data[i]->data, pSchema->bytes, num);
if (i == 0 || if (i == 0 ||
(functionIDs[i] != TSDB_FUNC_COUNT && (functionIDs[i] != TSDB_FUNC_COUNT &&
!isNull(srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes, pSchema->type)) || !isNull(srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes, pSchema->type)) ||
(functionIDs[i] == TSDB_FUNC_COUNT && (functionIDs[i] == TSDB_FUNC_COUNT &&
*(int64_t*)(srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes) != 0)) { *(int64_t*)(srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes) != 0)) {
assignVal(val1, srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes, pSchema->bytes, pSchema->type); assignVal(val1, srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes, pSchema->bytes, pSchema->type);
memcpy(*prevValues + tlen, srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes, pSchema->bytes); memcpy(*prevValues + tlen, srcData[i] + pInterpoInfo->rowIdx * pSchema->bytes, pSchema->bytes);
} else { // i > 0 and isNULL, do interpolation } else { // i > 0 and isNULL, do interpolation
@ -416,11 +406,11 @@ int32_t taosDoInterpoResult(SInterpolationInfo* pInterpoInfo, int16_t interpoTyp
/* set the tag value for final result */ /* set the tag value for final result */
setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, pModel->numOfCols - numOfTags, bufSize, setTagsValueInInterpolation(data, pTags, pModel, pInterpoInfo->order, pModel->numOfCols - numOfTags, bufSize,
num); num);
}
pInterpoInfo->startTimestamp += (nInterval * step); pInterpoInfo->startTimestamp += (nInterval * step);
pInterpoInfo->rowIdx += 1; pInterpoInfo->rowIdx += 1;
num += 1; num += 1;
}
if ((pInterpoInfo->rowIdx >= pInterpoInfo->numOfRawDataInRows && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) || if ((pInterpoInfo->rowIdx >= pInterpoInfo->numOfRawDataInRows && INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) ||
(pInterpoInfo->rowIdx < 0 && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) || num >= outputRows) { (pInterpoInfo->rowIdx < 0 && !INTERPOL_IS_ASC_INTERPOL(pInterpoInfo)) || num >= outputRows) {

28
src/util/src/tresultBuf.c
View File

@ -7,8 +7,8 @@
#define DEFAULT_INTERN_BUF_SIZE 16384L #define DEFAULT_INTERN_BUF_SIZE 16384L
int32_t createResultBuf(SQueryResultBuf** pResultBuf, int32_t size, int32_t rowSize) { int32_t createDiskbasedResultBuffer(SQueryDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize) {
SQueryResultBuf* pResBuf = calloc(1, sizeof(SQueryResultBuf)); SQueryDiskbasedResultBuf* pResBuf = calloc(1, sizeof(SQueryDiskbasedResultBuf));
pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / rowSize; pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_SIZE - sizeof(tFilePage)) / rowSize;
pResBuf->numOfPages = size; pResBuf->numOfPages = size;
@ -50,17 +50,17 @@ int32_t createResultBuf(SQueryResultBuf** pResultBuf, int32_t size, int32_t rowS
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
tFilePage* getResultBufferPageById(SQueryResultBuf* pResultBuf, int32_t id) { tFilePage* getResultBufferPageById(SQueryDiskbasedResultBuf* pResultBuf, int32_t id) {
assert(id < pResultBuf->numOfPages && id >= 0); assert(id < pResultBuf->numOfPages && id >= 0);
return (tFilePage*)(pResultBuf->pBuf + DEFAULT_INTERN_BUF_SIZE * id); return (tFilePage*)(pResultBuf->pBuf + DEFAULT_INTERN_BUF_SIZE * id);
} }
int32_t getNumOfResultBufGroupId(SQueryResultBuf* pResultBuf) { return taosNumElemsInHashTable(pResultBuf->idsTable); } int32_t getNumOfResultBufGroupId(SQueryDiskbasedResultBuf* pResultBuf) { return taosNumElemsInHashTable(pResultBuf->idsTable); }
int32_t getResBufSize(SQueryResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } int32_t getResBufSize(SQueryDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; }
static int32_t extendDiskFileSize(SQueryResultBuf* pResultBuf, int32_t numOfPages) { static int32_t extendDiskFileSize(SQueryDiskbasedResultBuf* pResultBuf, int32_t numOfPages) {
assert(pResultBuf->numOfPages * DEFAULT_INTERN_BUF_SIZE == pResultBuf->totalBufSize); assert(pResultBuf->numOfPages * DEFAULT_INTERN_BUF_SIZE == pResultBuf->totalBufSize);
int32_t ret = munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); int32_t ret = munmap(pResultBuf->pBuf, pResultBuf->totalBufSize);
@ -88,11 +88,11 @@ static int32_t extendDiskFileSize(SQueryResultBuf* pResultBuf, int32_t numOfPage
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool noMoreAvailablePages(SQueryResultBuf* pResultBuf) { static bool noMoreAvailablePages(SQueryDiskbasedResultBuf* pResultBuf) {
return (pResultBuf->allocateId == pResultBuf->numOfPages - 1); return (pResultBuf->allocateId == pResultBuf->numOfPages - 1);
} }
static int32_t getGroupIndex(SQueryResultBuf* pResultBuf, int32_t groupId) { static int32_t getGroupIndex(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId) {
assert(pResultBuf != NULL); assert(pResultBuf != NULL);
char* p = taosGetDataFromHashTable(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); char* p = taosGetDataFromHashTable(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t));
@ -106,7 +106,7 @@ static int32_t getGroupIndex(SQueryResultBuf* pResultBuf, int32_t groupId) {
return slot; return slot;
} }
static int32_t addNewGroupId(SQueryResultBuf* pResultBuf, int32_t groupId) { static int32_t addNewGroupId(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId) {
int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot
if (pResultBuf->numOfAllocGroupIds <= num) { if (pResultBuf->numOfAllocGroupIds <= num) {
@ -148,7 +148,7 @@ static int32_t doRegisterId(SIDList* pList, int32_t id) {
return 0; return 0;
} }
static void registerPageId(SQueryResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { static void registerPageId(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
int32_t slot = getGroupIndex(pResultBuf, groupId); int32_t slot = getGroupIndex(pResultBuf, groupId);
if (slot < 0) { if (slot < 0) {
slot = addNewGroupId(pResultBuf, groupId); slot = addNewGroupId(pResultBuf, groupId);
@ -158,7 +158,7 @@ static void registerPageId(SQueryResultBuf* pResultBuf, int32_t groupId, int32_t
doRegisterId(pList, pageId); doRegisterId(pList, pageId);
} }
tFilePage* getNewDataBuf(SQueryResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { tFilePage* getNewDataBuf(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
if (noMoreAvailablePages(pResultBuf)) { if (noMoreAvailablePages(pResultBuf)) {
if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) {
return NULL; return NULL;
@ -177,9 +177,9 @@ tFilePage* getNewDataBuf(SQueryResultBuf* pResultBuf, int32_t groupId, int32_t*
return page; return page;
} }
int32_t getNumOfRowsPerPage(SQueryResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } int32_t getNumOfRowsPerPage(SQueryDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
SIDList getDataBufPagesIdList(SQueryResultBuf* pResultBuf, int32_t groupId) { SIDList getDataBufPagesIdList(SQueryDiskbasedResultBuf* pResultBuf, int32_t groupId) {
SIDList list = {0}; SIDList list = {0};
int32_t slot = getGroupIndex(pResultBuf, groupId); int32_t slot = getGroupIndex(pResultBuf, groupId);
if (slot < 0) { if (slot < 0) {
@ -189,7 +189,7 @@ SIDList getDataBufPagesIdList(SQueryResultBuf* pResultBuf, int32_t groupId) {
} }
} }
void destroyResultBuf(SQueryResultBuf* pResultBuf) { void destroyResultBuf(SQueryDiskbasedResultBuf* pResultBuf) {
if (pResultBuf == NULL) { if (pResultBuf == NULL) {
return; return;
} }