xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/shm

This commit is contained in:
Shengliang Guan 2022-04-05 15:38:01 +08:00
parent 69aa4546f5 ca3a0b3e02
commit 13770e971e
60 changed files with 2095 additions and 952 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
include/common/tcommon.h
View File

@ -53,12 +53,12 @@ typedef struct {
typedef struct SColumnDataAgg {
int16_t colId;
int64_t sum;
int64_t max;
int64_t min;
int16_t maxIndex;
int16_t minIndex;
int16_t numOfNull;
int64_t sum;
int64_t max;
int64_t min;
} SColumnDataAgg;
typedef struct SDataBlockInfo {
@ -193,20 +193,19 @@ typedef struct SColumn {
uint8_t scale;
} SColumn;
typedef struct SLimit {
int64_t limit;
int64_t offset;
} SLimit;
typedef struct SOrder {
uint32_t order;
SColumn col;
} SOrder;
typedef struct SGroupbyExpr {
SArray* columnInfo; // SArray<SColIndex>, group by columns information
bool groupbyTag; // group by tag or column
} SGroupbyExpr;
typedef struct STableBlockDistInfo {
uint16_t rowSize;
uint16_t numOfFiles;
uint32_t numOfTables;
uint64_t totalSize;
uint64_t totalRows;
int32_t maxRows;
int32_t minRows;
int32_t firstSeekTimeUs;
uint32_t numOfRowsInMemTable;
uint32_t numOfSmallBlocks;
SArray *dataBlockInfos;
} STableBlockDistInfo;
enum {
FUNC_PARAM_TYPE_VALUE = 0x1,
@ -241,15 +240,6 @@ typedef struct SExprInfo {
struct tExprNode* pExpr;
} SExprInfo;
typedef struct SStateWindow {
SColumn col;
} SStateWindow;
typedef struct SSessionWindow {
int64_t gap; // gap between two session window(in microseconds)
SColumn col;
} SSessionWindow;
#define QUERY_ASC_FORWARD_STEP 1
#define QUERY_DESC_FORWARD_STEP -1

5
include/common/tdataformat.h
View File

@ -135,8 +135,8 @@ typedef struct {
#define TD_VTYPE_PARTS 4 // 8 bits / TD_VTYPE_BITS = 4
#define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder
#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS))
#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8))
#define TD_BITMAP_BYTES(cnt) (ceil((double)(cnt) / TD_VTYPE_PARTS))
#define TD_BIT_TO_BYTES(cnt) (ceil((double)(cnt) / 8))
int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
@ -365,6 +365,7 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche
}
#endif
// ----------------- Data column structure
// SDataCol arrangement: data => bitmap => dataOffset
typedef struct SDataCol {
int8_t type; // column type
uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM

14
include/common/trow.h
View File

@ -148,15 +148,15 @@ typedef struct {
typedef struct {
// basic info
int8_t rowType;
int16_t sver;
schema_ver_t sver;
STSRow *pBuf;
// extended info
int32_t flen;
int16_t nBoundCols;
int16_t nCols;
int16_t nBitmaps;
int16_t nBoundBitmaps;
col_id_t nBoundCols;
col_id_t nCols;
col_id_t nBitmaps;
col_id_t nBoundBitmaps;
int32_t offset;
void *pBitmap;
void *pOffset;
@ -448,9 +448,9 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t
}
#ifdef TD_SUPPORT_BITMAP
// the primary TS key is stored separatedly
pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
if (nBoundCols > 0) {
pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
} else {
pBuilder->nBoundBitmaps = 0;
}

9
include/libs/function/function.h
View File

@ -52,6 +52,11 @@ typedef struct SFuncExecFuncs {
FExecFinalize finalize;
} SFuncExecFuncs;
typedef struct SFileBlockInfo {
int32_t numBlocksOfStep;
} SFileBlockInfo;
#define TSDB_BLOCK_DIST_STEP_ROWS 8
#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results
#define FUNCTION_TYPE_SCALAR 1
@ -101,10 +106,6 @@ typedef struct SFuncExecFuncs {
#define FUNCTION_DERIVATIVE 32
#define FUNCTION_BLKINFO 33
#define FUNCTION_HISTOGRAM 34
#define FUNCTION_HLL 35
#define FUNCTION_MODE 36
#define FUNCTION_SAMPLE 37
#define FUNCTION_COV 38

10
include/libs/function/functionMgt.h
View File

@ -72,10 +72,15 @@ typedef enum EFunctionType {
FUNCTION_TYPE_ATAN,
// string function
FUNCTION_TYPE_CHAR_LENGTH = 1500,
FUNCTION_TYPE_LENGTH = 1500,
FUNCTION_TYPE_CHAR_LENGTH,
FUNCTION_TYPE_CONCAT,
FUNCTION_TYPE_CONCAT_WS,
FUNCTION_TYPE_LENGTH,
FUNCTION_TYPE_LOWER,
FUNCTION_TYPE_UPPER,
FUNCTION_TYPE_LTRIM,
FUNCTION_TYPE_RTRIM,
FUNCTION_TYPE_SUBSTR,
// conversion function
FUNCTION_TYPE_CAST = 2000,
@ -127,6 +132,7 @@ bool fmIsStringFunc(int32_t funcId);
bool fmIsDatetimeFunc(int32_t funcId);
bool fmIsTimelineFunc(int32_t funcId);
bool fmIsTimeorderFunc(int32_t funcId);
bool fmIsPseudoColumnFunc(int32_t funcId);
bool fmIsWindowPseudoColumnFunc(int32_t funcId);
bool fmIsWindowClauseFunc(int32_t funcId);

16
include/libs/nodes/nodes.h
View File

@ -213,16 +213,16 @@ typedef enum EDealRes {
} EDealRes;
typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext);
void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext);
void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext);
void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext);
void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext);
void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext);
void nodesWalkExprs(SNodeList* pList, FNodeWalker walker, void* pContext);
void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext);
void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext);
typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext);
void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext);
void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext);
void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext);
void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext);
void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
bool nodesEqualNode(const SNodeptr a, const SNodeptr b);

1
include/libs/nodes/querynodes.h
View File

@ -79,6 +79,7 @@ typedef struct SValueNode {
char* literal;
bool isDuration;
bool translate;
bool genByCalc;
union {
bool b;
int64_t i;

12
include/libs/scalar/scalar.h
View File

@ -42,6 +42,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);
int32_t vectorGetConvertType(int32_t type1, int32_t type2);
int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut);
/* Math functions */
int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
@ -58,6 +59,17 @@ int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
/* String functions */
int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

15
source/client/src/clientMain.c
View File

@ -264,8 +264,17 @@ int *taos_fetch_lengths(TAOS_RES *res) {
}
TAOS_ROW *taos_result_block(TAOS_RES *res) {
// TODO
SRequestObj* pRequest = (SRequestObj*) res;
if (pRequest == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return NULL;
}
if (taos_is_update_query(res)) {
return NULL;
}
return &pRequest->body.resInfo.row;
}
// todo intergrate with tDataTypes
@ -313,7 +322,9 @@ int taos_affected_rows(TAOS_RES *res) {
return pResInfo->numOfRows;
}
int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; }
int taos_result_precision(TAOS_RES *res) {
return TSDB_TIME_PRECISION_MILLI;
}
int taos_select_db(TAOS *taos, const char *db) {
STscObj *pObj = (STscObj *)taos;

67
source/client/src/tmq.c
View File

@ -79,6 +79,7 @@ struct tmq_t {
tmq_commit_cb* commit_cb;
int32_t nextTopicIdx;
int8_t epStatus;
int32_t epSkipCnt;
int32_t waitingRequest;
int32_t readyRequest;
SArray* clientTopics; // SArray<SMqClientTopic>
@ -107,6 +108,7 @@ typedef struct {
// connection info
int32_t vgId;
int32_t vgStatus;
int64_t skipCnt;
SEpSet epSet;
} SMqClientVg;
@ -138,6 +140,7 @@ typedef struct {
tmq_t* tmq;
SMqClientVg* pVg;
int32_t epoch;
int32_t vgId;
tsem_t rspSem;
tmq_message_t** msg;
int32_t sync;
@ -313,6 +316,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
pTmq->waitingRequest = 0;
pTmq->readyRequest = 0;
pTmq->epStatus = 0;
pTmq->epSkipCnt = 0;
// set conf
strcpy(pTmq->clientId, conf->clientId);
strcpy(pTmq->groupId, conf->groupId);
@ -348,6 +352,8 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->epoch = 0;
pTmq->waitingRequest = 0;
pTmq->readyRequest = 0;
pTmq->epStatus = 0;
pTmq->epSkipCnt = 0;
// set conf
strcpy(pTmq->clientId, conf->clientId);
strcpy(pTmq->groupId, conf->groupId);
@ -834,7 +840,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqClientVg* pVg = pParam->pVg;
tmq_t* tmq = pParam->tmq;
if (code != 0) {
tscWarn("msg discard, code:%x", code);
tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code);
goto CREATE_MSG_FAIL;
}
@ -842,13 +848,13 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
if (msgEpoch < tmqEpoch) {
/*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
tsem_post(&tmq->rspSem);
tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
/*tsem_post(&tmq->rspSem);*/
tscWarn("discard rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
return 0;
}
if (msgEpoch != tmqEpoch) {
tscWarn("mismatch rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
} else {
atomic_sub_fetch_32(&tmq->waitingRequest, 1);
}
@ -892,29 +898,29 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
}
#endif
tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset,
tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, pRsp->msg.reqOffset,
pRsp->msg.rspOffset);
pRsp->vg = pParam->pVg;
taosWriteQitem(tmq->mqueue, pRsp);
atomic_add_fetch_32(&tmq->readyRequest, 1);
tsem_post(&tmq->rspSem);
/*tsem_post(&tmq->rspSem);*/
return 0;
CREATE_MSG_FAIL:
if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
}
tsem_post(&tmq->rspSem);
/*tsem_post(&tmq->rspSem);*/
return code;
}
bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
/*printf("call update ep %d\n", epoch);*/
tscDebug("tmq update ep epoch %d to epoch %d", tmq->epoch, epoch);
bool set = false;
int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
char vgKey[TSDB_TOPIC_FNAME_LEN + 22];
tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, topicNumGet);
SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic));
if (newTopics == NULL) {
return false;
@ -932,17 +938,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
taosHashClear(pHash);
topic.topicName = strdup(pTopicEp->topic);
tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName);
int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
for (int32_t j = 0; j < topicNumCur; j++) {
// find old topic
SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j);
if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) {
int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs);
tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur);
if (vgNumCur == 0) break;
for (int32_t k = 0; k < vgNumCur; k++) {
SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k);
sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId);
tscDebug("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);
tscDebug("consumer %ld epoch %d vg %d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey);
taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
}
break;
@ -956,18 +964,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
int64_t offset = pVgEp->offset;
tscDebug("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);
tscDebug("consumer %ld epoch %d vg %d offset og to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset);
if (pOffset != NULL) {
offset = *pOffset;
tscDebug("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);
tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey);
}
tscDebug("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);
tscDebug("consumer %ld epoch %d vg %d offset set to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset);
SMqClientVg clientVg = {
.pollCnt = 0,
.currentOffset = offset,
.vgId = pVgEp->vgId,
.epSet = pVgEp->epSet,
.vgStatus = TMQ_VG_STATUS__IDLE,
.skipCnt = 0,
};
taosArrayPush(topic.vgs, &clientVg);
set = true;
@ -984,9 +993,8 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
tmq_t* tmq = pParam->tmq;
tscDebug("consumer %ld recv ep", tmq->consumerId);
if (code != 0) {
tscError("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync);
goto END;
}
@ -995,6 +1003,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
// Epoch will only increase when received newer epoch ep msg
SMqRspHead* head = pMsg->pData;
int32_t epoch = atomic_load_32(&tmq->epoch);
tscDebug("consumer %ld recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch);
if (head->epoch <= epoch) {
goto END;
}
@ -1019,7 +1028,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp);
taosWriteQitem(tmq->mqueue, pRsp);
tsem_post(&tmq->rspSem);
/*tsem_post(&tmq->rspSem);*/
}
END:
@ -1033,9 +1042,11 @@ END:
int32_t tmqAskEp(tmq_t* tmq, bool sync) {
int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
if (epStatus == 1) {
tscDebug("consumer %ld skip ask ep", tmq->consumerId);
return 0;
int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
if (epSkipCnt < 5000) return 0;
}
atomic_store_32(&tmq->epSkipCnt, 0);
int32_t tlen = sizeof(SMqCMGetSubEpReq);
SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen);
if (req == NULL) {
@ -1221,24 +1232,34 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
if (vgStatus != TMQ_VG_STATUS__IDLE) {
tscDebug("consumer %ld skip vg %d", tmq->consumerId, pVg->vgId);
int64_t skipCnt = atomic_add_fetch_64(&pVg->skipCnt, 1);
tscDebug("consumer %ld epoch %d skip vg %d skip cnt %ld", tmq->consumerId, tmq->epoch, pVg->vgId, skipCnt);
continue;
#if 0
if (skipCnt < 30000) {
continue;
} else {
tscDebug("consumer %ld skip vg %d skip too much reset", tmq->consumerId, pVg->vgId);
}
#endif
}
atomic_store_64(&pVg->skipCnt, 0);
SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
tsem_post(&tmq->rspSem);
/*tsem_post(&tmq->rspSem);*/
return -1;
}
SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
if (pParam == NULL) {
taosMemoryFree(pReq);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
tsem_post(&tmq->rspSem);
/*tsem_post(&tmq->rspSem);*/
return -1;
}
pParam->tmq = tmq;
pParam->pVg = pVg;
pParam->vgId = pVg->vgId;
pParam->epoch = tmq->epoch;
pParam->sync = 0;
@ -1247,7 +1268,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
taosMemoryFree(pReq);
taosMemoryFree(pParam);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
tsem_post(&tmq->rspSem);
/*tsem_post(&tmq->rspSem);*/
return -1;
}
@ -1265,7 +1286,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
int64_t transporterId = 0;
/*printf("send poll\n");*/
atomic_add_fetch_32(&tmq->waitingRequest, 1);
tscDebug("consumer %ld send poll: vg %d, req offset %ld", tmq->consumerId, pVg->vgId, pVg->currentOffset);
tscDebug("consumer %ld send poll: vg %d, epoch %d, req offset %ld", tmq->consumerId, pVg->vgId, tmq->epoch, pVg->currentOffset);
/*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
pVg->pollCnt++;
@ -1379,7 +1400,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
tmqAskEp(tmq, false);
tmqPollImpl(tmq, blocking_time);
tsem_wait(&tmq->rspSem);
/*tsem_wait(&tmq->rspSem);*/
rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
if (rspMsg) {

26
source/common/src/tdataformat.c
View File

@ -33,7 +33,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
spaceNeeded += (int)nBitmapBytes;
// TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more
// TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered.
spaceNeeded += TYPE_BYTES[pCol->type];
// spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove
// the additional space
#endif
if (pCol->spaceSize < spaceNeeded) {
@ -47,6 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
}
}
#ifdef TD_SUPPORT_BITMAP
if (IS_VAR_DATA_TYPE(pCol->type)) {
pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
pCol->dataOff = POINTER_SHIFT(pCol->pBitmap, nBitmapBytes);
@ -306,7 +308,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row)
bool isNEleNull(SDataCol *pCol, int nEle) {
if (isAllRowsNull(pCol)) return true;
for (int i = 0; i < nEle; i++) {
for (int i = 0; i < nEle; ++i) {
if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false;
}
return true;
@ -327,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
if (IS_VAR_DATA_TYPE(pCol->type)) {
pCol->len = 0;
for (int i = 0; i < nEle; i++) {
for (int i = 0; i < nEle; ++i) {
dataColSetNullAt(pCol, i);
}
} else {
@ -343,7 +345,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) {
// char *tptr = (char *)(pCol->pData);
VarDataOffsetT offset = 0;
for (int i = 0; i < nEle; i++) {
for (int i = 0; i < nEle; ++i) {
pCol->dataOff[i] = offset;
offset += varDataTLen(tptr);
tptr = POINTER_SHIFT(tptr, varDataTLen(tptr));
@ -371,6 +373,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
tdFreeDataCols(pCols);
return NULL;
}
#if 0 // no need as calloc used
int i;
for (i = 0; i < maxCols; i++) {
pCols->cols[i].spaceSize = 0;
@ -378,6 +381,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
pCols->cols[i].pData = NULL;
pCols->cols[i].dataOff = NULL;
}
#endif
}
return pCols;
@ -391,17 +395,21 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
if (ptr == NULL) return -1;
pCols->cols = ptr;
for (i = oldMaxCols; i < pCols->maxCols; i++) {
for (i = oldMaxCols; i < pCols->maxCols; ++i) {
pCols->cols[i].pData = NULL;
pCols->cols[i].dataOff = NULL;
pCols->cols[i].pBitmap = NULL;
pCols->cols[i].spaceSize = 0;
}
}
#if 0
tdResetDataCols(pCols); // redundant loop to reset len/blen to 0, already reset in following dataColInit(...)
#endif
tdResetDataCols(pCols);
pCols->numOfRows = 0;
pCols->numOfCols = schemaNCols(pSchema);
for (i = 0; i < schemaNCols(pSchema); i++) {
for (i = 0; i < schemaNCols(pSchema); ++i) {
dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints);
}
@ -413,7 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
if (pCols) {
if (pCols->cols) {
int maxCols = pCols->maxCols;
for (i = 0; i < maxCols; i++) {
for (i = 0; i < maxCols; ++i) {
SDataCol *pCol = &pCols->cols[i];
taosMemoryFreeClear(pCol->pData);
}
@ -464,7 +472,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
void tdResetDataCols(SDataCols *pCols) {
if (pCols != NULL) {
pCols->numOfRows = 0;
for (int i = 0; i < pCols->maxCols; i++) {
for (int i = 0; i < pCols->maxCols; ++i) {
dataColReset(pCols->cols + i);
}
}

6
source/common/src/tmsg.c
View File

@ -367,7 +367,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx));
for (col_id_t i = 0; i < pReq->stbCfg.nCols; i++) {
for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
@ -376,7 +376,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
}
buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols);
pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
@ -408,7 +408,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx));
for (col_id_t i = 0; i < pReq->ntbCfg.nCols; i++) {
for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma);
buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);

2
source/common/src/trow.c
View File

@ -503,7 +503,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
}
if (!TD_COL_ROWS_NORM(pRet->cols + i)) {
int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints);
int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows);
memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes);
}
}

5
source/dnode/mgmt/vm/vmHandle.c
View File

@ -30,6 +30,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2;
pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
pCfg->tsdbCfg.retentions = pCreate->pRetensions;
pCfg->metaCfg.lruSize = pCreate->cacheBlockSize;
pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod;
pCfg->walCfg.level = pCreate->walLevel;
@ -70,6 +71,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId);
if (pVnode != NULL) {
tFreeSCreateVnodeReq(&createReq);
dDebug("vgId:%d, already exist", createReq.vgId);
vmReleaseVnode(pMgmt, pVnode);
terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED;
@ -88,12 +90,14 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
vnodeCfg.dbId = wrapperCfg.dbUid;
SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg);
if (pImpl == NULL) {
tFreeSCreateVnodeReq(&createReq);
dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
return -1;
}
int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl);
if (code != 0) {
tFreeSCreateVnodeReq(&createReq);
dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
vnodeClose(pImpl);
vnodeDestroy(wrapperCfg.path);
@ -103,6 +107,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
code = vmWriteVnodesToFile(pMgmt);
if (code != 0) {
tFreeSCreateVnodeReq(&createReq);
vnodeClose(pImpl);
vnodeDestroy(wrapperCfg.path);
terrno = code;

4
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -160,10 +160,8 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
pOldConsumer->epoch++;
// TODO handle update
/*taosWLockLatch(&pOldConsumer->lock);*/
atomic_add_fetch_32(&pOldConsumer->epoch, 1);
/*taosWUnLockLatch(&pOldConsumer->lock);*/
return 0;

3
source/dnode/mnode/impl/src/mndDb.c
View File

@ -69,7 +69,8 @@ void mndCleanupDb(SMnode *pMnode) {}
static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE);
SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER,
sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + TSDB_DB_RESERVE_SIZE);
if (pRaw == NULL) goto DB_ENCODE_OVER;
int32_t dataPos = 0;

3
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -471,6 +471,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
consumerEp.consumerId = -1;
consumerEp.epSet = plan->execNode.epSet;
consumerEp.vgId = plan->execNode.nodeId;
mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId);
int32_t msgLen;
if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
sdbRelease(pSdb, pVgroup);

24
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -212,19 +212,24 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
return -1;
}
//TODO add lock
ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0);
int32_t serverEpoch = pConsumer->epoch;
// TODO
int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);
mTrace("try to get sub ep, old val: %d", hbStatus);
mDebug("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, serverEpoch, hbStatus);
atomic_store_32(&pConsumer->hbStatus, 0);
/*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/
/*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/
/*sdbWrite(pMnode->pSdb, pConsumerRaw);*/
strcpy(rsp.cgroup, pReq->cgroup);
if (epoch != pConsumer->epoch) {
mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, pConsumer->epoch);
if (epoch != serverEpoch) {
mInfo("send new assignment to consumer %ld, consumer epoch %d, server epoch %d", pConsumer->consumerId, epoch, serverEpoch);
mDebug("consumer %ld try r lock", consumerId);
taosRLockLatch(&pConsumer->lock);
mDebug("consumer %ld r locked", consumerId);
SArray *pTopics = pConsumer->currentTopics;
int32_t sz = taosArrayGetSize(pTopics);
rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp));
@ -238,7 +243,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
if (consumerId == pSubConsumer->consumerId) {
int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
mInfo("topic %s has %d vg", topicName, pConsumer->epoch);
mInfo("topic %s has %d vg", topicName, serverEpoch);
SMqSubTopicEp topicEp;
strcpy(topicEp.topic, topicName);
topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp));
@ -264,6 +269,8 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
}
mndReleaseSubscribe(pMnode, pSub);
}
taosRUnLockLatch(&pConsumer->lock);
mDebug("consumer %ld r unlock", consumerId);
}
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp);
void *buf = rpcMallocCont(tlen);
@ -272,7 +279,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
return -1;
}
((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP;
((SMqRspHead *)buf)->epoch = pConsumer->epoch;
((SMqRspHead *)buf)->epoch = serverEpoch;
((SMqRspHead *)buf)->consumerId = pConsumer->consumerId;
void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
@ -395,7 +402,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key);
taosMemoryFreeClear(pRebSub->key);
mInfo("mq rebalance subscription: %s", pSub->key);
mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, (int32_t)taosArrayGetSize(pSub->unassignedVg));
// remove lost consumer
for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) {
@ -442,6 +449,9 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
}
SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
mDebug("consumer %ld try w lock", pRebConsumer->consumerId);
taosWLockLatch(&pRebConsumer->lock);
mDebug("consumer %ld w locked", pRebConsumer->consumerId);
int32_t status = atomic_load_32(&pRebConsumer->status);
if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb ||
(vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) ||
@ -462,6 +472,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
mndTransAppendCommitlog(pTrans, pConsumerRaw);
}
taosWUnLockLatch(&pRebConsumer->lock);
mDebug("consumer %ld w unlock", pRebConsumer->consumerId);
mndReleaseConsumer(pMnode, pRebConsumer);
}

5
source/dnode/vnode/inc/tsdb.h
View File

@ -55,13 +55,14 @@ typedef struct STsdbCfg {
int8_t precision;
int8_t update;
int8_t compression;
uint64_t lruCacheSize;
int32_t daysPerFile;
int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock;
int32_t keep;
int32_t keep1;
int32_t keep2;
uint64_t lruCacheSize;
SArray *retentions;
} STsdbCfg;
// query condition to build multi-table data block iterator
@ -171,6 +172,8 @@ tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo
tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef);
int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo);
bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle);
/**

8
source/dnode/vnode/src/inc/tsdbReadImpl.h
View File

@ -112,10 +112,10 @@ typedef struct {
#else
typedef struct {
int16_t colId;
uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
uint8_t reserve : 7;
uint8_t type;
int32_t len;
uint16_t type : 6;
uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input)
uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
uint32_t len : 31; // data length + bitmap length
uint32_t offset;
} SBlockColV0;

35
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -1281,7 +1281,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest);
int32_t keyLen = 0;
int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite);
int32_t tBitmaps = 0;
// int32_t tBitmaps = 0;
for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) {
// All not NULL columns finish
@ -1297,7 +1297,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
#ifdef TD_SUPPORT_BITMAP
int32_t tBitmaps = 0;
int32_t tBitmapsLen = 0;
if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
tBitmaps = nBitmaps;
#if 0
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
tBitmaps = nBitmaps;
tlen += tBitmaps;
@ -1305,16 +1308,17 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
tlen += tBitmaps * TYPE_BYTES[pDataCol->type];
}
#endif
// move bitmap parts ahead
// TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove
memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps);
// memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps);
}
#endif
void *tptr;
void *tptr, *bptr;
// Make room
if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) {
if (tsdbMakeRoom(ppBuf, lsize + tlen + tBitmaps + 2 * COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) {
return -1;
}
pBlockData = (SBlockData *)(*ppBuf);
@ -1327,23 +1331,44 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
// Compress or just copy
if (pCfg->compression) {
#if 0
flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr,
tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf,
tlen + COMP_OVERFLOW_BYTES);
#endif
flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr,
tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf,
tlen + COMP_OVERFLOW_BYTES);
if (tBitmaps > 0) {
bptr = POINTER_SHIFT(pBlockData, lsize + flen);
tBitmapsLen =
tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES,
pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES);
TASSERT((tBitmapsLen > 0) && (tBitmapsLen <= (tBitmaps + COMP_OVERFLOW_BYTES)));
flen += tBitmapsLen;
}
} else {
flen = tlen;
memcpy(tptr, pDataCol->pData, flen);
if (tBitmaps > 0) {
bptr = POINTER_SHIFT(pBlockData, lsize + flen);
memcpy(bptr, pDataCol->pBitmap, tBitmaps);
tBitmapsLen = tBitmaps;
flen += tBitmapsLen;
}
}
// Add checksum
ASSERT(flen > 0);
ASSERT(tBitmapsLen <= 1024);
flen += sizeof(TSCKSUM);
taosCalcChecksumAppend(0, (uint8_t *)tptr, flen);
tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)));
if (ncol != 0) {
tsdbSetBlockColOffset(pBlockCol, toffset);
pBlockCol->len = flen;
pBlockCol->len = flen; // data + bitmaps
pBlockCol->blen = tBitmapsLen;
++tcol;
} else {
keyLen = flen;

90
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -14,7 +14,7 @@
*/
#include "tsdbDef.h"
#include <tdatablock.h>
#include "tdatablock.h"
#include "os.h"
#include "talgo.h"
#include "tcompare.h"
@ -31,6 +31,7 @@
#include "tlosertree.h"
#include "tsdbDef.h"
#include "tmsg.h"
#include "tsdbCommit.h"
#define EXTRA_BYTES 2
#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
@ -209,34 +210,34 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load
return pLocalIdList;
}
//int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) {
// STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
//
// int64_t rows = 0;
// STsdbMemTable* pMemTable = pTsdbReadHandle->pMemTable;
// if (pMemTable == NULL) { return rows; }
//
//// STableData* pMem = NULL;
//// STableData* pIMem = NULL;
//
//// SMemTable* pMemT = pMemRef->snapshot.mem;
//// SMemTable* pIMemT = pMemRef->snapshot.imem;
//
// size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
// for (int32_t i = 0; i < size; ++i) {
// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
//
//// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) {
//// pMem = pMemT->tData[pCheckInfo->tableId];
//// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0;
//// }
//// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) {
//// pIMem = pIMemT->tData[pCheckInfo->tableId];
//// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0;
//// }
int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) {
STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle;
int64_t rows = 0;
STsdbMemTable* pMemTable = NULL;//pTsdbReadHandle->pMemTable;
if (pMemTable == NULL) { return rows; }
// STableData* pMem = NULL;
// STableData* pIMem = NULL;
// SMemTable* pMemT = pMemRef->snapshot.mem;
// SMemTable* pIMemT = pMemRef->snapshot.imem;
size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
for (int32_t i = 0; i < size; ++i) {
STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) {
// pMem = pMemT->tData[pCheckInfo->tableId];
// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0;
// }
// return rows;
//}
// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) {
// pIMem = pIMemT->tData[pCheckInfo->tableId];
// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0;
// }
}
return rows;
}
static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList) {
size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList);
@ -2261,12 +2262,13 @@ static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) {
cur->mixBlock = false;
cur->blockCompleted = false;
}
#if 0
int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTableBlockInfo) {
int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo* pTableBlockInfo) {
STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle;
pTableBlockInfo->totalSize = 0;
pTableBlockInfo->totalRows = 0;
STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb);
// find the start data block in file
@ -2284,9 +2286,11 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa
int32_t code = TSDB_CODE_SUCCESS;
int32_t numOfBlocks = 0;
int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
int defaultRows = 4096;//TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
STimeWindow win = TSWINDOW_INITIALIZER;
bool ascTraverse = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
while (true) {
numOfBlocks = 0;
tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
@ -2299,8 +2303,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa
tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
// current file are not overlapped with query time window, ignore remain files
if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) ||
(!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) {
if ((ascTraverse && win.skey > pTsdbReadHandle->window.ekey) || (!ascTraverse && win.ekey < pTsdbReadHandle->window.ekey)) {
tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb));
tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pTsdbReadHandle,
pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
@ -2342,19 +2345,26 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTa
int32_t numOfRows = pBlock[j].numOfRows;
pTableBlockInfo->totalRows += numOfRows;
if (numOfRows > pTableBlockInfo->maxRows) pTableBlockInfo->maxRows = numOfRows;
if (numOfRows < pTableBlockInfo->minRows) pTableBlockInfo->minRows = numOfRows;
if (numOfRows < defaultRows) pTableBlockInfo->numOfSmallBlocks+=1;
int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS;
SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex);
blockInfo->numBlocksOfStep++;
if (numOfRows > pTableBlockInfo->maxRows) {
pTableBlockInfo->maxRows = numOfRows;
}
if (numOfRows < pTableBlockInfo->minRows) {
pTableBlockInfo->minRows = numOfRows;
}
if (numOfRows < defaultRows) {
pTableBlockInfo->numOfSmallBlocks += 1;
}
// int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS;
// SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex);
// blockInfo->numBlocksOfStep++;
}
}
}
return code;
}
#endif
static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exists) {
STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb);

71
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
View File

@ -21,9 +21,8 @@ static void tsdbResetReadTable(SReadH *pReadh);
static void tsdbResetReadFile(SReadH *pReadh);
static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock);
static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols);
static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer,
int bufferSize);
static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp,
int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize);
static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
int numOfColIds);
static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
@ -548,7 +547,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
if (dcol != 0 && ccol >= pBlockData->numOfCols) {
// Set current column as NULL and forward
dataColReset(pDataCol);
dcol++;
++dcol;
continue;
}
@ -567,9 +566,11 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
TD_SET_COL_ROWS_NORM(pDataCol);
}
int32_t tBitmaps = 0;
// int32_t tBitmaps = 0;
int32_t tLenBitmap = 0;
if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
tLenBitmap = nBitmaps;
#if 0
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
tBitmaps = nBitmaps;
tLenBitmap = tBitmaps;
@ -577,17 +578,18 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
}
#endif
}
if (tcolId == pDataCol->colId) {
if (pBlock->algorithm == TWO_STAGE_COMP) {
int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
int zsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES;
if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1;
}
if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm,
pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
(int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlockCol->blen,
pBlock->algorithm, pBlock->numOfRows, tLenBitmap, pDataCols->maxPoints,
TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u",
TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
return -1;
@ -609,9 +611,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
return 0;
}
static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer,
int bufferSize) {
static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp,
int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize) {
if (!taosCheckChecksumWhole((uint8_t *)content, len)) {
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return -1;
@ -623,21 +624,41 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
if (comp) {
// Need to decompress
int tlen =
(*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows + numOfBitmaps,
(*(tDataTypes[pDataCol->type].decompFunc))(content, len - bitmapLen - sizeof(TSCKSUM), numOfRows,
pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
if (tlen <= 0) {
tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
len, comp, numOfRows, maxPoints, bufferSize);
tsdbError(
"Failed to decompress column data, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
(int32_t)(len - bitmapLen - sizeof(TSCKSUM)), comp, numOfRows, maxPoints, bufferSize);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return -1;
}
pDataCol->len = tlen;
if (numOfBitmaps > 0) {
tlen = tsDecompressTinyint(POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen, numOfBitmaps,
pDataCol->pBitmap, pDataCol->spaceSize, comp, buffer, bufferSize);
if (tlen <= 0) {
tsdbError(
"Failed to decompress column bitmap, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d "
"bufferSize:%d",
bitmapLen, comp, numOfBitmaps, maxPoints, bufferSize);
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
return -1;
}
// pDataCol->blen = tlen;
}
} else {
// No need to decompress, just memcpy it
pDataCol->len = len - sizeof(TSCKSUM);
pDataCol->len = len - bitmapLen - sizeof(TSCKSUM);
memcpy(pDataCol->pData, content, pDataCol->len);
if (numOfBitmaps > 0) {
// pDataCol->blen = bitmapLen;
memcpy(pDataCol->pBitmap, POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen);
}
}
#if 0
if (lenOfBitmaps > 0) {
pDataCol->len -= lenOfBitmaps;
@ -653,7 +674,10 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
} else if (IS_VAR_DATA_TYPE(pDataCol->type)) {
dataColSetOffset(pDataCol, numOfRows);
}
#endif
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
dataColSetOffset(pDataCol, numOfRows);
}
return 0;
}
@ -740,14 +764,16 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) {
ASSERT(pDataCol->colId == pBlockCol->colId);
STsdb * pRepo = TSDB_READ_REPO(pReadh);
STsdb *pRepo = TSDB_READ_REPO(pReadh);
STsdbCfg *pCfg = REPO_CFG(pRepo);
int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows);
int32_t tBitmaps = 0;
// int32_t tBitmaps = 0;
int32_t tLenBitmap = 0;
if (!TD_COL_ROWS_NORM(pBlockCol)) {
tLenBitmap = nBitmaps;
#if 0
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
tBitmaps = nBitmaps;
tLenBitmap = tBitmaps;
@ -755,9 +781,10 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
}
#endif
}
int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES;
int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES;
if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
@ -785,8 +812,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
return -1;
}
if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows,
tBitmaps, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf,
if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlockCol->blen, pBlock->algorithm,
pBlock->numOfRows, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf,
(int32_t)taosTSizeof(pReadh->pCBuf)) < 0) {
tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
pBlockCol->colId, offset);

23
source/libs/executor/inc/executorimpl.h
View File

@ -128,6 +128,11 @@ typedef struct {
int64_t sumRunTimes;
} SOperatorProfResult;
typedef struct SLimit {
int64_t limit;
int64_t offset;
} SLimit;
typedef struct STaskCostInfo {
int64_t created;
int64_t start;
@ -163,6 +168,11 @@ typedef struct SOperatorCostInfo {
uint64_t execCost;
} SOperatorCostInfo;
typedef struct SOrder {
uint32_t order;
SColumn col;
} SOrder;
// The basic query information extracted from the SQueryInfo tree to support the
// execution of query in a data node.
typedef struct STaskAttr {
@ -196,7 +206,6 @@ typedef struct STaskAttr {
STimeWindow window;
SInterval interval;
SSessionWindow sw;
int16_t precision;
int16_t numOfOutput;
int16_t fillType;
@ -206,13 +215,8 @@ typedef struct STaskAttr {
int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query.
int32_t maxTableColumnWidth;
int32_t tagLen; // tag value length of current query
SGroupbyExpr* pGroupbyExpr;
SExprInfo* pExpr1;
SExprInfo* pExpr2;
int32_t numOfExpr2;
SExprInfo* pExpr3;
int32_t numOfExpr3;
SColumnInfo* tableCols;
SColumnInfo* tagColList;
@ -220,8 +224,6 @@ typedef struct STaskAttr {
int64_t* fillVal;
SSingleColumnFilterInfo* pFilterInfo;
// SFilterInfo *pFilters;
void* tsdb;
STableGroupInfo tableGroupInfo; // table <tid, last_key> list SArray<STableKeyInfo>
int32_t vgId;
@ -384,7 +386,7 @@ typedef struct SExchangeInfo {
} SExchangeInfo;
typedef struct STableScanInfo {
void* pTsdbReadHandle;
void* dataReader;
int32_t numOfBlocks; // extract basic running information.
int32_t numOfSkipped;
int32_t numOfBlockStatis;
@ -644,12 +646,13 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock,
int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
int32_t numOfOutput);
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,

508
source/libs/executor/src/executorimpl.c
View File

@ -249,7 +249,6 @@ static void operatorDummyCloseFn(void* param, int32_t numOfCols) {}
static int32_t doCopyToSDataBlock(SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType,
SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset);
static int32_t getGroupbyColumnIndex(SGroupbyExpr* pGroupbyExpr, SSDataBlock* pDataBlock);
static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type,
int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo,
SAggSupporter* pAggSup);
@ -287,44 +286,6 @@ static int compareRowData(const void* a, const void* b, const void* userData) {
return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0;
}
static void sortGroupResByOrderList(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRuntimeEnv,
SSDataBlock* pDataBlock) {
SArray* columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr);
size_t size = taosArrayGetSize(columnOrderList);
taosArrayDestroy(columnOrderList);
if (size <= 0) {
return;
}
int32_t orderId = pRuntimeEnv->pQueryAttr->order.col.colId;
if (orderId <= 0) {
return;
}
bool found = false;
int16_t dataOffset = 0;
for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) {
SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pDataBlock->pDataBlock, j);
if (orderId == j) {
found = true;
break;
}
dataOffset += pColInfoData->info.bytes;
}
if (found == false) {
return;
}
int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resSchema.type;
SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)};
taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support);
}
// setup the output buffer for each operator
SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) {
int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
@ -375,17 +336,6 @@ static bool isSelectivityWithTagsQuery(SqlFunctionCtx* pCtx, int32_t numOfOutput
// return (numOfSelectivity > 0 && hasTags);
}
static bool isProjQuery(STaskAttr* pQueryAttr) {
for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
int32_t functId = getExprFunctionId(&pQueryAttr->pExpr1[i]);
if (functId != FUNCTION_PRJ && functId != FUNCTION_TAGPRJ) {
return false;
}
}
return true;
}
static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) {
if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) ||
pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
@ -1411,15 +1361,10 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo,
}
static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos,
int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols,
STimeWindow* win) {
STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);
int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) {
bool ascQuery = true;
TSKEY curTs = tsCols[pos];
TSKEY lastTs = *(TSKEY*)pRuntimeEnv->prevRow[0];
TSKEY lastTs = 0;//*(TSKEY*)pRuntimeEnv->prevRow[0];
// lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed.
// start exactly from this point, no need to do interpolation
@ -1434,27 +1379,24 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlF
return true;
}
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step];
doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key,
RESULT_ROW_START_INTERP);
doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP);
return true;
}
static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex,
SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey,
STimeWindow* win) {
STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
int32_t order = TSDB_ORDER_ASC;
int32_t numOfOutput = pOperatorInfo->numOfOutput;
TSKEY actualEndKey = tsCols[endRowIndex];
TSKEY key = QUERY_IS_ASC_QUERY(pQueryAttr) ? win->ekey : win->skey;
TSKEY key = order ? win->ekey : win->skey;
// not ended in current data block, do not invoke interpolation
if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQueryAttr)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQueryAttr))) {
if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) {
setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP);
return false;
}
@ -1465,7 +1407,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun
return true;
}
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);
int32_t nextRowIndex = endRowIndex + step;
assert(nextRowIndex >= 0);
@ -1667,10 +1609,9 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv;
int32_t numOfOutput = pOperatorInfo->numOfOutput;
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr);
int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order);
bool ascQuery = true;
TSKEY* tsCols = NULL;
if (pSDataBlock->pDataBlock != NULL) {
@ -1683,7 +1624,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
int32_t startPos = ascQuery ? 0 : (pSDataBlock->info.rows - 1);
TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery);
STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr);
STimeWindow win = {0};//getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr);
bool masterScan = IS_MAIN_SCAN(pRuntimeEnv);
SResultRow* pResult = NULL;
@ -1699,7 +1640,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
TSKEY ekey = reviseWindowEkey(pQueryAttr, &win);
TSKEY ekey = 0;//reviseWindowEkey(pQueryAttr, &win);
// forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey,
// binarySearchForKey, true);
@ -1713,31 +1654,31 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
// startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey,
// prevEndPos);
if (startPos < 0) {
if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) {
int32_t code =
setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult,
tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset);
if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
}
startPos = pSDataBlock->info.rows - 1;
// if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) {
// int32_t code =
// setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult,
// tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset);
// if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
// longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
// }
//
// startPos = pSDataBlock->info.rows - 1;
// window start(end) key interpolation
// doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos,
// forwardStep); doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos,
// forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput);
}
// }
break;
}
setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
}
if (pQueryAttr->timeWindowInterpo) {
int32_t rowIndex = ascQuery ? (pSDataBlock->info.rows - 1) : 0;
// if (pQueryAttr->timeWindowInterpo) {
// int32_t rowIndex = ascQuery ? (pSDataBlock->info.rows - 1) : 0;
// saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex);
}
// }
// updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey);
}
@ -2023,28 +1964,6 @@ static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo* binfo, int32_t numOfCo
return TSDB_CODE_SUCCESS;
}
static int32_t getGroupbyColumnIndex(SGroupbyExpr* pGroupbyExpr, SSDataBlock* pDataBlock) {
size_t num = taosArrayGetSize(pGroupbyExpr->columnInfo);
for (int32_t k = 0; k < num; ++k) {
SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k);
if (TSDB_COL_IS_TAG(pColIndex->flag)) {
continue;
}
int32_t colId = pColIndex->colId;
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i);
if (pColInfo->info.colId == colId) {
return i;
}
}
}
assert(0);
return -1;
}
static bool functionNeedToExecute(SqlFunctionCtx* pCtx) {
struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
@ -2264,64 +2183,6 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
return NULL;
}
static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv* pRuntimeEnv, int32_t numOfTables, SArray* pOperator,
void* merger) {
// qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv));
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
pRuntimeEnv->prevGroupId = INT32_MIN;
pRuntimeEnv->pQueryAttr = pQueryAttr;
pRuntimeEnv->pResultRowHashTable =
taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
pRuntimeEnv->pResultRowListSet =
taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
pRuntimeEnv->keyBuf = taosMemoryMalloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES);
// pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv));
pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell));
pRuntimeEnv->prevRow = taosMemoryMalloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize);
pRuntimeEnv->tagVal = taosMemoryMalloc(pQueryAttr->tagLen);
// NOTE: pTableCheckInfo need to update the query time range and the lastKey info
pRuntimeEnv->pTableRetrieveTsMap =
taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
// pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput);
if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL ||
pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) {
goto _clean;
}
if (pQueryAttr->numOfCols) {
char* start = POINTER_BYTES * pQueryAttr->numOfCols + (char*)pRuntimeEnv->prevRow;
pRuntimeEnv->prevRow[0] = start;
for (int32_t i = 1; i < pQueryAttr->numOfCols; ++i) {
pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQueryAttr->tableCols[i - 1].bytes;
}
if (pQueryAttr->tableCols[0].type == TSDB_DATA_TYPE_TIMESTAMP) {
*(int64_t*)pRuntimeEnv->prevRow[0] = INT64_MIN;
}
}
// qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv));
// group by normal column, sliding window query, interval query are handled by interval query processor
// interval (down sampling operation)
return TSDB_CODE_SUCCESS;
_clean:
// destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput);
taosMemoryFreeClear(pRuntimeEnv->pResultRowHashTable);
taosMemoryFreeClear(pRuntimeEnv->keyBuf);
taosMemoryFreeClear(pRuntimeEnv->prevRow);
taosMemoryFreeClear(pRuntimeEnv->tagVal);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) {
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
@ -2400,17 +2261,6 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_Q
// return false;
//}
static bool isFirstLastRowQuery(STaskAttr* pQueryAttr) {
for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
int32_t functionID = getExprFunctionId(&pQueryAttr->pExpr1[i]);
if (functionID == FUNCTION_LAST_ROW) {
return true;
}
}
return false;
}
static bool isCachedLastQuery(STaskAttr* pQueryAttr) {
for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]);
@ -2493,18 +2343,6 @@ static bool onlyOneQueryType(STaskAttr* pQueryAttr, int32_t functId, int32_t fun
return true;
}
static bool onlyFirstQuery(STaskAttr* pQueryAttr) {
return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST);
}
static bool onlyLastQuery(STaskAttr* pQueryAttr) {
return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST);
}
static bool notContainSessionOrStateWindow(STaskAttr* pQueryAttr) {
return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow);
}
static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) {
bool hasFirstLastFunc = false;
bool hasOtherFunc = false;
@ -3006,7 +2844,7 @@ int32_t loadDataBlock(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo,
*status = BLK_DATA_ALL_NEEDED;
SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL);
SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
if (pCols == NULL) {
return terrno;
}
@ -3899,76 +3737,6 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S
return 0;
}
// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function.
void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput,
SExprInfo* pExprInfo) {
#if 0
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
int32_t numOfExprs = pQueryAttr->numOfOutput;
for(int32_t i = 0; i < numOfExprs; ++i) {
SExprInfo* pExprInfo1 = &(pExprInfo[i]);
if (pExprInfo1->base.functionId != FUNCTION_STDDEV_DST) {
continue;
}
SExprBasicInfo* pExpr = &pExprInfo1->base;
pCtx[i].param[0].arr = NULL;
pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int
// TODO use hash to speedup this loop
int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult);
for (int32_t j = 0; j < numOfGroup; ++j) {
SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j);
if (pQueryAttr->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQueryAttr->tagLen) == 0) {
int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult);
for (int32_t k = 0; k < numOfCols; ++k) {
SStddevInterResult* pres = taosArrayGet(p->pResult, k);
if (pres->info.colId == pExpr->colInfo.colId) {
pCtx[i].param[0].arr = pres->pResult;
break;
}
}
}
}
}
#endif
}
void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput,
SExprInfo* pExpr, char* val, int16_t bytes) {
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
#if 0
int32_t numOfExprs = pQueryAttr->numOfOutput;
for(int32_t i = 0; i < numOfExprs; ++i) {
SExprBasicInfo* pExpr1 = &pExpr[i].base;
if (pExpr1->functionId != FUNCTION_STDDEV_DST) {
continue;
}
pCtx[i].param[0].arr = NULL;
pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int
// TODO use hash to speedup this loop
int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult);
for (int32_t j = 0; j < numOfGroup; ++j) {
SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j);
if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) {
int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult);
for (int32_t k = 0; k < numOfCols; ++k) {
SStddevInterResult* pres = taosArrayGet(p->pResult, k);
if (pres->info.colId == pExpr1->colInfo.colId) {
pCtx[i].param[0].arr = pres->pResult;
break;
}
}
}
}
}
#endif
}
/*
* There are two cases to handle:
*
@ -4679,13 +4447,13 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
*newgroup = false;
while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) {
while (tsdbNextDataBlock(pTableScanInfo->dataReader)) {
if (isTaskKilled(pOperator->pTaskInfo)) {
longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
pTableScanInfo->numOfBlocks += 1;
tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info);
tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info);
// todo opt
// if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) {
@ -4722,7 +4490,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
// The read handle is not initialized yet, since no qualified tables exists
if (pTableScanInfo->pTsdbReadHandle == NULL) {
if (pTableScanInfo->dataReader == NULL) {
return NULL;
}
@ -4750,11 +4518,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
pTableScanInfo->scanFlag = REPEAT_SCAN;
// if (pTaskInfo->pTsBuf) {
// bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf);
// assert(ret);
// }
//
if (pResultRowInfo->size > 0) {
pResultRowInfo->curPos = 0;
}
@ -4790,44 +4553,44 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) {
STableScanInfo* pTableScanInfo = pOperator->info;
*newgroup = false;
#if 0
STableBlockDist tableBlockDist = {0};
tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables;
STableBlockDistInfo tableBlockDist = {0};
tableBlockDist.numOfTables = 1; // TODO set the correct number of tables
int32_t numRowSteps = TSDB_DEFAULT_MAX_ROW_FBLOCK / TSDB_BLOCK_DIST_STEP_ROWS;
if (TSDB_DEFAULT_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) {
++numRowSteps;
}
tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo));
taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps);
tableBlockDist.maxRows = INT_MIN;
tableBlockDist.minRows = INT_MAX;
tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist);
tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle);
tsdbGetFileBlocksDistInfo(pTableScanInfo->dataReader, &tableBlockDist);
tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->dataReader);
SSDataBlock* pBlock = &pTableScanInfo->block;
pBlock->info.rows = 1;
pBlock->info.numOfCols = 1;
SBufferWriter bw = tbufInitWriter(NULL, false);
blockDistInfoToBinary(&tableBlockDist, &bw);
// SBufferWriter bw = tbufInitWriter(NULL, false);
// blockDistInfoToBinary(&tableBlockDist, &bw);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0);
int32_t len = (int32_t) tbufTell(&bw);
pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t));
// int32_t len = (int32_t) tbufTell(&bw);
// pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t));
// *(int32_t*) pColInfo->pData = len;
// memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len);
//
// tbufCloseWriter(&bw);
*(int32_t*) pColInfo->pData = len;
memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len);
tbufCloseWriter(&bw);
SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0);
pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0);
// SArray* g = GET_TABLEGROUP(pOperator->, 0);
// pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0);
pOperator->status = OP_EXEC_DONE;
return pBlock;
#endif
}
static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) {
@ -5472,7 +5235,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
}
pInfo->pFilterNode = pCondition;
pInfo->pTsdbReadHandle = pTsdbReadHandle;
pInfo->dataReader = pTsdbReadHandle;
pInfo->times = repeatTime;
pInfo->reverseTimes = reverseTime;
pInfo->order = order;
@ -5494,7 +5257,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) {
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
pInfo->pTsdbReadHandle = pTsdbReadHandle;
pInfo->dataReader = pTsdbReadHandle;
pInfo->times = 1;
pInfo->reverseTimes = 0;
pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
@ -5515,32 +5278,42 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim
return pOperator;
}
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) {
SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo) {
STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
goto _error;
}
pInfo->pTsdbReadHandle = pTsdbReadHandle;
pInfo->dataReader = dataReader;
pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
SColumnInfoData infoData = {{0}};
SColumnInfoData infoData = {0};
infoData.info.type = TSDB_DATA_TYPE_BINARY;
infoData.info.bytes = 1024;
infoData.info.colId = 0;
taosArrayPush(pInfo->block.pDataBlock, &infoData);
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
pOperator->name = "TableBlockInfoScanOperator";
pOperator->name = "DataBlockInfoScanOperator";
// pOperator->operatorType = OP_TableBlockInfoScan;
pOperator->blockingOptr = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
// pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols;
pOperator->_openFn = operatorDummyOpenFn;
pOperator->getNextFn = doBlockInfoScan;
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
return pOperator;
_error:
taosMemoryFreeClear(pInfo);
taosMemoryFreeClear(pOperator);
return NULL;
}
SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList,
SArray* pTableIdList, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pResBlock, SArray* pColList, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) {
SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -5589,6 +5362,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
pOperator->getNextFn = doStreamBlockScan;
pOperator->closeFn = operatorDummyCloseFn;
pOperator->pTaskInfo = pTaskInfo;
return pOperator;
}
@ -5704,7 +5478,7 @@ void getDBNameFromCondition(SNode* pCondition, char* dbName) {
return;
}
nodesWalkNode(pCondition, getDBNameFromConditionWalker, dbName);
nodesWalkExpr(pCondition, getDBNameFromConditionWalker, dbName);
}
static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
@ -5922,80 +5696,6 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
return pOperator;
}
SArray* getOrderCheckColumns(STaskAttr* pQuery) {
int32_t numOfCols = (pQuery->pGroupbyExpr == NULL) ? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo);
SArray* pOrderColumns = NULL;
if (numOfCols > 0) {
pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo);
} else {
pOrderColumns = taosArrayInit(4, sizeof(SColIndex));
}
if (pQuery->interval.interval > 0) {
if (pOrderColumns == NULL) {
pOrderColumns = taosArrayInit(1, sizeof(SColIndex));
}
SColIndex colIndex = {.colIndex = 0, .colId = 0, .flag = TSDB_COL_NORMAL};
taosArrayPush(pOrderColumns, &colIndex);
}
{
numOfCols = (int32_t)taosArrayGetSize(pOrderColumns);
for (int32_t i = 0; i < numOfCols; ++i) {
SColIndex* index = taosArrayGet(pOrderColumns, i);
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
SExprBasicInfo* pExpr = &pQuery->pExpr1[j].base;
int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]);
if (index->colId == pExpr->pParam[0].pCol->colId &&
(functionId == FUNCTION_PRJ || functionId == FUNCTION_TAG || functionId == FUNCTION_TS)) {
index->colIndex = j;
index->colId = pExpr->resSchema.colId;
}
}
}
}
return pOrderColumns;
}
SArray* getResultGroupCheckColumns(STaskAttr* pQuery) {
int32_t numOfCols = (pQuery->pGroupbyExpr == NULL) ? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo);
SArray* pOrderColumns = NULL;
if (numOfCols > 0) {
pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo);
} else {
pOrderColumns = taosArrayInit(4, sizeof(SColIndex));
}
for (int32_t i = 0; i < numOfCols; ++i) {
SColIndex* index = taosArrayGet(pOrderColumns, i);
bool found = false;
for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
SExprBasicInfo* pExpr = &pQuery->pExpr1[j].base;
int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]);
// FUNCTION_TAG_DUMMY function needs to be ignored
// if (index->colId == pExpr->pColumns->info.colId &&
// ((TSDB_COL_IS_TAG(pExpr->pColumns->flag) && functionId == FUNCTION_TAG) ||
// (TSDB_COL_IS_NORMAL_COL(pExpr->pColumns->flag) && functionId == FUNCTION_PRJ))) {
// index->colIndex = j;
// index->colId = pExpr->resSchema.colId;
// found = true;
// break;
// }
}
assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput);
}
return pOrderColumns;
}
static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, const char* pKey);
static void cleanupAggSup(SAggSupporter* pAggSup);
@ -7044,9 +6744,6 @@ static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgr
return pIntervalInfo->binfo.pRes;
}
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
int32_t order = pQueryAttr->order.order;
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
@ -7062,14 +6759,14 @@ static SSDataBlock* doAllSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgr
STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current;
// setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput);
setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order);
// setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order);
setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey);
hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex);
}
pOperator->status = OP_RES_TO_RETURN;
pQueryAttr->order.order = order; // TODO : restore the order
// pQueryAttr->order.order = order; // TODO : restore the order
doCloseAllTimeWindow(pRuntimeEnv);
setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
@ -8153,7 +7850,6 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) {
int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]);
if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
assert(pQueryAttr->numOfOutput == 1);
SExprInfo* pExprInfo = &pOperator->pExpr[0];
@ -8936,8 +8632,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
return pList;
}
int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo,
uint64_t queryId, uint64_t taskId) {
int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) {
int32_t code = 0;
if (tableType == TSDB_SUPER_TABLE) {
code = tsdbQuerySTableByTagCond(metaHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId);
@ -9041,43 +8736,6 @@ static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SCol
return TSDB_CODE_SUCCESS;
}
static void doUpdateExprColumnIndex(STaskAttr* pQueryAttr) {
assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL);
for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) {
SExprBasicInfo* pSqlExprMsg = &pQueryAttr->pExpr1[k].base;
// if (pSqlExprMsg->functionId == FUNCTION_ARITHM) {
// continue;
// }
// todo opt performance
SColIndex* pColIndex = NULL; /*&pSqlExprMsg->colInfo;*/
if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) {
int32_t f = 0;
for (f = 0; f < pQueryAttr->numOfCols; ++f) {
if (pColIndex->colId == pQueryAttr->tableCols[f].colId) {
pColIndex->colIndex = f;
break;
}
}
assert(f < pQueryAttr->numOfCols);
} else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) {
// do nothing for user-defined constant value result columns
} else {
int32_t f = 0;
for (f = 0; f < pQueryAttr->numOfTags; ++f) {
if (pColIndex->colId == pQueryAttr->tagColList[f].colId) {
pColIndex->colIndex = f;
break;
}
}
assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX);
}
}
}
void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) {
const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512);
@ -9087,16 +8745,16 @@ void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) {
const float THRESHOLD_RATIO = 0.85f;
if (isProjQuery(pQueryAttr)) {
int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize;
if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) {
numOfRes = MIN_ROWS_FOR_PRJ_QUERY;
}
pResultInfo->capacity = numOfRes;
} else { // in case of non-prj query, a smaller output buffer will be used.
pResultInfo->capacity = DEFAULT_MIN_ROWS;
}
// if (isProjQuery(pQueryAttr)) {
// int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize;
// if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) {
// numOfRes = MIN_ROWS_FOR_PRJ_QUERY;
// }
//
// pResultInfo->capacity = numOfRes;
// } else { // in case of non-prj query, a smaller output buffer will be used.
// pResultInfo->capacity = DEFAULT_MIN_ROWS;
// }
pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO);
pResultInfo->totalRows = 0;

5
source/libs/function/inc/builtinsimpl.h
View File

@ -38,9 +38,14 @@ void minFunction(SqlFunctionCtx* pCtx);
void maxFunction(SqlFunctionCtx *pCtx);
bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
void stddevFunction(SqlFunctionCtx* pCtx);
void stddevFinalize(SqlFunctionCtx* pCtx);
bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
void percentileFunction(SqlFunctionCtx *pCtx);
bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
void firstFunction(SqlFunctionCtx *pCtx);
void lastFunction(SqlFunctionCtx *pCtx);

154
source/libs/function/src/builtins.c
View File

@ -68,9 +68,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.classification = FUNC_MGT_AGG_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = getStddevFuncEnv,
.initFunc = maxFunctionSetup,
.processFunc = maxFunction,
.finalizeFunc = functionFinalize
.initFunc = stddevFunctionSetup,
.processFunc = stddevFunction,
.finalizeFunc = stddevFinalize
},
{
.name = "percentile",
@ -282,6 +282,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = atanFunction,
.finalizeFunc = NULL
},
{
.name = "length",
.type = FUNCTION_TYPE_LENGTH,
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = lengthFunction,
.finalizeFunc = NULL
},
{
.name = "char_length",
.type = FUNCTION_TYPE_CHAR_LENGTH,
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = charLengthFunction,
.finalizeFunc = NULL
},
{
.name = "concat",
.type = FUNCTION_TYPE_CONCAT,
@ -289,7 +309,67 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.sprocessFunc = concatFunction,
.finalizeFunc = NULL
},
{
.name = "concat_ws",
.type = FUNCTION_TYPE_CONCAT_WS,
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = concatWsFunction,
.finalizeFunc = NULL
},
{
.name = "lower",
.type = FUNCTION_TYPE_LOWER,
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = lowerFunction,
.finalizeFunc = NULL
},
{
.name = "upper",
.type = FUNCTION_TYPE_UPPER,
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = upperFunction,
.finalizeFunc = NULL
},
{
.name = "ltrim",
.type = FUNCTION_TYPE_LTRIM,
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = ltrimFunction,
.finalizeFunc = NULL
},
{
.name = "rtrim",
.type = FUNCTION_TYPE_RTRIM,
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = rtrimFunction,
.finalizeFunc = NULL
},
{
.name = "substr",
.type = FUNCTION_TYPE_SUBSTR,
.classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC,
.checkFunc = stubCheckAndGetResultType,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = substrFunction,
.finalizeFunc = NULL
},
{
@ -409,12 +489,6 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType };
break;
}
case FUNCTION_TYPE_CONCAT:
case FUNCTION_TYPE_ROWTS:
case FUNCTION_TYPE_TBNAME: {
// todo
break;
}
case FUNCTION_TYPE_QENDTS:
case FUNCTION_TYPE_QSTARTTS:
@ -434,6 +508,7 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
break;
}
case FUNCTION_TYPE_STDDEV:
case FUNCTION_TYPE_SIN:
case FUNCTION_TYPE_COS:
case FUNCTION_TYPE_TAN:
@ -446,6 +521,65 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE };
break;
}
case FUNCTION_TYPE_LENGTH:
case FUNCTION_TYPE_CHAR_LENGTH: {
pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_SMALLINT].bytes, .type = TSDB_DATA_TYPE_SMALLINT };
break;
}
case FUNCTION_TYPE_CONCAT:
case FUNCTION_TYPE_CONCAT_WS: {
int32_t paraType, paraBytes = 0;
for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) {
SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i);
paraBytes += pParam->node.resType.bytes;
paraType = pParam->node.resType.type;
}
pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType };
break;
//int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0;
//int32_t firstParamIndex = 0;
//if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) {
// firstParamIndex = 1;
// SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0);
// sepBytes = pSep->node.resType.type;
//}
//for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) {
// SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i);
// int32_t paraType = pParam->node.resType.type;
// if (i == firstParamIndex) {
// paraTypeFirst = paraType;
// }
// if (paraType != paraTypeFirst) {
// return TSDB_CODE_FAILED;
// }
// //TODO: for constants also needs numOfRows
// totalBytes += pParam->node.resType.bytes;
//}
////TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100.
//totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100;
//pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst };
//break;
}
case FUNCTION_TYPE_LOWER:
case FUNCTION_TYPE_UPPER:
case FUNCTION_TYPE_LTRIM:
case FUNCTION_TYPE_RTRIM:
case FUNCTION_TYPE_SUBSTR: {
SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0);
int32_t paraType = pParam->node.resType.type;
int32_t paraBytes = pParam->node.resType.bytes;
pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType };
break;
}
case FUNCTION_TYPE_ROWTS:
case FUNCTION_TYPE_TBNAME: {
// todo
break;
}
case FUNCTION_TYPE_NOW:
// todo
break;

212
source/libs/function/src/builtinsimpl.c
View File

@ -14,6 +14,7 @@
*/
#include "builtinsimpl.h"
#include "tpercentile.h"
#include "querynodes.h"
#include "taggfunction.h"
#include "tdatablock.h"
@ -453,6 +454,7 @@ bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
}
typedef struct SStddevRes {
double result;
int64_t count;
union {double quadraticDSum; int64_t quadraticISum;};
union {double dsum; int64_t isum;};
@ -463,23 +465,64 @@ bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
return true;
}
bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
if (!functionSetup(pCtx, pResultInfo)) {
return false;
}
SStddevRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo);
memset(pRes, 0, sizeof(SStddevRes));
return true;
}
void stddevFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElem = 0;
// Only the pre-computing information loaded and actual data does not loaded
SInputColumnInfoData* pInput = &pCtx->input;
SColumnDataAgg *pAgg = pInput->pColumnDataAgg[0];
SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0];
int32_t type = pInput->pData[0]->info.type;
SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
// } else { // computing based on the true data block
// computing based on the true data block
SColumnInfoData* pCol = pInput->pData[0];
int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows;
switch(type) {
switch (type) {
case TSDB_DATA_TYPE_TINYINT: {
int8_t* plist = (int8_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pStddevRes->count += 1;
pStddevRes->isum += plist[i];
pStddevRes->quadraticISum += plist[i] * plist[i];
}
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
int16_t* plist = (int16_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pStddevRes->count += 1;
pStddevRes->isum += plist[i];
pStddevRes->quadraticISum += plist[i] * plist[i];
}
break;
}
case TSDB_DATA_TYPE_INT: {
int32_t* plist = (int32_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
@ -487,11 +530,61 @@ void stddevFunction(SqlFunctionCtx* pCtx) {
continue;
}
numOfElem += 1;
pStddevRes->count += 1;
pStddevRes->isum += plist[i];
pStddevRes->quadraticISum += plist[i] * plist[i];
}
break;
}
case TSDB_DATA_TYPE_BIGINT: {
int64_t* plist = (int64_t*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pStddevRes->count += 1;
pStddevRes->isum += plist[i];
pStddevRes->quadraticISum += plist[i] * plist[i];
}
break;
}
case TSDB_DATA_TYPE_FLOAT: {
float* plist = (float*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pStddevRes->count += 1;
pStddevRes->isum += plist[i];
pStddevRes->quadraticISum += plist[i] * plist[i];
}
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
double* plist = (double*)pCol->pData;
for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
continue;
}
numOfElem += 1;
pStddevRes->count += 1;
pStddevRes->isum += plist[i];
pStddevRes->quadraticISum += plist[i] * plist[i];
}
break;
}
default:
break;
}
@ -503,11 +596,122 @@ void stddevFinalize(SqlFunctionCtx* pCtx) {
functionFinalize(pCtx);
SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
double res = pStddevRes->quadraticISum/pStddevRes->count - (pStddevRes->isum / pStddevRes->count) * (pStddevRes->isum / pStddevRes->count);
double avg = pStddevRes->isum / ((double) pStddevRes->count);
pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg);
}
typedef struct SPercentileInfo {
tMemBucket *pMemBucket;
int32_t stage;
double minval;
double maxval;
int64_t numOfElems;
} SPercentileInfo;
bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
pEnv->calcMemSize = sizeof(SPercentileInfo);
return true;
}
bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
if (!functionSetup(pCtx, pResultInfo)) {
return false;
}
// in the first round, get the min-max value of all involved data
SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
pInfo->numOfElems = 0;
return true;
}
void percentileFunction(SqlFunctionCtx *pCtx) {
int32_t notNullElems = 0;
#if 0
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) {
pInfo->stage += 1;
// all data are null, set it completed
if (pInfo->numOfElems == 0) {
pResInfo->complete = true;
return;
} else {
pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
}
}
// the first stage, only acquire the min/max value
if (pInfo->stage == 0) {
if (pCtx->preAggVals.isSet) {
double tmin = 0.0, tmax = 0.0;
if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) {
tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min);
tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max);
} else if (IS_FLOAT_TYPE(pCtx->inputType)) {
tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min);
tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max);
} else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) {
tmin = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.min);
tmax = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.max);
} else {
assert(true);
}
if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) {
SET_DOUBLE_VAL(&pInfo->minval, tmin);
}
if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) {
SET_DOUBLE_VAL(&pInfo->maxval, tmax);
}
pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
} else {
for (int32_t i = 0; i < pCtx->size; ++i) {
char *data = GET_INPUT_DATA(pCtx, i);
if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
continue;
}
double v = 0;
GET_TYPED_DATA(v, double, pCtx->inputType, data);
if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
SET_DOUBLE_VAL(&pInfo->minval, v);
}
if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
SET_DOUBLE_VAL(&pInfo->maxval, v);
}
pInfo->numOfElems += 1;
}
}
return;
}
// the second stage, calculate the true percentile value
for (int32_t i = 0; i < pCtx->size; ++i) {
char *data = GET_INPUT_DATA(pCtx, i);
if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
continue;
}
notNullElems += 1;
tMemBucketPut(pInfo->pMemBucket, data, 1);
}
SET_VAL(pCtx, notNullElems, 1);
pResInfo->hasResult = DATA_SET_FLAG;
#endif
}
bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0);

4
source/libs/function/src/functionMgt.c
View File

@ -104,6 +104,10 @@ bool fmIsScalarFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC);
}
bool fmIsPseudoColumnFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC);
}
bool fmIsWindowPseudoColumnFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC);
}

36
source/libs/function/src/taggfunction.c
View File

@ -176,26 +176,6 @@ typedef struct SResPair {
double avg;
} SResPair;
#define TSDB_BLOCK_DIST_STEP_ROWS 16
typedef struct STableBlockDist {
uint16_t rowSize;
uint16_t numOfFiles;
uint32_t numOfTables;
uint64_t totalSize;
uint64_t totalRows;
int32_t maxRows;
int32_t minRows;
int32_t firstSeekTimeUs;
uint32_t numOfRowsInMemTable;
uint32_t numOfSmallBlocks;
SArray *dataBlockInfos;
} STableBlockDist;
typedef struct SFileBlockInfo {
int32_t numBlocksOfStep;
} SFileBlockInfo;
void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) {
pCell->initialized = false;
}
@ -3984,7 +3964,7 @@ static void irate_function(SqlFunctionCtx *pCtx) {
}
}
static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) {
static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDistInfo* pDist) {
SBufferReader br = tbufInitReader(data, len, false);
pDist->numOfTables = tbufReadUint32(&br);
@ -4024,7 +4004,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi
static void blockInfo_func(SqlFunctionCtx* pCtx) {
SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo);
int32_t len = *(int32_t*) pCtx->pInput;
blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist);
@ -4036,8 +4016,8 @@ static void blockInfo_func(SqlFunctionCtx* pCtx) {
//pResInfo->hasResult = DATA_SET_FLAG;
}
static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDist* pSrc) {
STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDistInfo* pSrc) {
STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo);
assert(pDist != NULL && pSrc != NULL);
pDist->numOfTables += pSrc->numOfTables;
@ -4071,7 +4051,7 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock
}
void block_func_merge(SqlFunctionCtx* pCtx) {
STableBlockDist info = {0};
STableBlockDistInfo info = {0};
int32_t len = *(int32_t*) pCtx->pInput;
blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info);
SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
@ -4082,7 +4062,7 @@ void block_func_merge(SqlFunctionCtx* pCtx) {
//pResInfo->hasResult = DATA_SET_FLAG;
}
void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents,
void getPercentiles(STableBlockDistInfo *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents,
double* percents, int32_t* percentiles) {
if (totalBlocks == 0) {
for (int32_t i = 0; i < numOfPercents; ++i) {
@ -4117,7 +4097,7 @@ void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32
}
}
void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) {
void generateBlockDistResult(STableBlockDistInfo *pTableBlockDist, char* result) {
if (pTableBlockDist == NULL) {
return;
}
@ -4178,7 +4158,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) {
void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) {
SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo);
pDist->rowSize = (uint16_t)pCtx->param[0].i;
generateBlockDistResult(pDist, pCtx->pOutput);

9
source/libs/index/src/indexFst.c
View File

@ -642,6 +642,9 @@ static const char* fstNodeState(FstNode* node) {
}
void fstNodeDestroy(FstNode* node) {
if (node == NULL) {
return;
}
fstSliceDestroy(&node->data);
taosMemoryFree(node);
}
@ -1247,7 +1250,10 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
// autState = sws->aut->accept(preState, b);
autState = automFuncs[aut->type].accept(aut, preState, b);
taosArrayPush(sws->inp, &b);
StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState};
node = NULL;
taosArrayPush(sws->stack, &s);
out += trn.out;
node = fstGetNode(sws->fst, trn.addr);
@ -1271,6 +1277,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
return true;
}
}
fstNodeDestroy(node);
uint32_t sz = taosArrayGetSize(sws->stack);
if (sz != 0) {
StreamState* s = taosArrayGet(sws->stack, sz - 1);

19
source/libs/index/test/fstTest.cc
View File

@ -480,6 +480,15 @@ void checkFstCheckIteratorRange2() {
assert(result.size() == 4);
automCtxDestroy(ctx);
}
{
// range search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
// [b, e)
m->SearchRange(ctx, "bb", GE, "ed", LT, result);
assert(result.size() == 3);
automCtxDestroy(ctx);
}
{
// range search
std::vector<uint64_t> result;
@ -635,11 +644,11 @@ int main(int argc, char* argv[]) {
// path suid colName ver
// iterTFileReader(argv[1], argv[2], argv[3], argv[4]);
//}
// checkFstCheckIterator1();
// checkFstCheckIterator2();
// checkFstCheckIteratorPrefix();
// checkFstCheckIteratorRange1();
// checkFstCheckIteratorRange2();
checkFstCheckIterator1();
checkFstCheckIterator2();
checkFstCheckIteratorPrefix();
checkFstCheckIteratorRange1();
checkFstCheckIteratorRange2();
checkFstCheckIteratorRange3();
// checkFstLongTerm();
// checkFstPrefixSearch();

25
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -113,36 +113,13 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
}
static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
COPY_ALL_SCALAR_FIELDS;
exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
COPY_CHAR_POINT_FIELD(literal);
COPY_SCALAR_FIELD(isDuration);
COPY_SCALAR_FIELD(translate);
if (!pSrc->translate) {
return (SNode*)pDst;
}
switch (pSrc->node.resType.type) {
case TSDB_DATA_TYPE_NULL:
break;
case TSDB_DATA_TYPE_BOOL:
COPY_SCALAR_FIELD(datum.b);
break;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
COPY_SCALAR_FIELD(datum.i);
break;
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
COPY_SCALAR_FIELD(datum.u);
break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
COPY_SCALAR_FIELD(datum.d);
break;
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:

7
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -1491,6 +1491,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkValueGenByCalc = "GenByCalc";
static const char* jkValueLiteral = "Literal";
static const char* jkValueDuration = "Duration";
static const char* jkValueTranslate = "Translate";
@ -1544,6 +1545,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) {
int32_t code = exprNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkValueGenByCalc, pNode->genByCalc);
}
if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) {
code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal);
}
if (TSDB_CODE_SUCCESS == code) {
@ -1614,6 +1618,9 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkValueGenByCalc, &pNode->genByCalc);
}
if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) {
code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal);
}
if (TSDB_CODE_SUCCESS == code) {

48
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -138,19 +138,19 @@ static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalke
return DEAL_RES_CONTINUE;
}
void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext) {
void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext) {
(void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext);
}
void nodesWalkList(SNodeList* pNodeList, FNodeWalker walker, void* pContext) {
void nodesWalkExprs(SNodeList* pNodeList, FNodeWalker walker, void* pContext) {
(void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext);
}
void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) {
void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) {
(void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext);
}
void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) {
void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) {
(void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext);
}
@ -267,19 +267,19 @@ static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRe
return DEAL_RES_CONTINUE;
}
void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext) {
void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext) {
(void)rewriteNode(pNode, TRAVERSAL_PREORDER, rewriter, pContext);
}
void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext) {
void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext) {
(void)rewriteList(pList, TRAVERSAL_PREORDER, rewriter, pContext);
}
void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) {
void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) {
(void)rewriteNode(pNode, TRAVERSAL_POSTORDER, rewriter, pContext);
}
void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) {
void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) {
(void)rewriteList(pList, TRAVERSAL_POSTORDER, rewriter, pContext);
}
@ -290,20 +290,20 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa
switch (clause) {
case SQL_CLAUSE_FROM:
nodesWalkNode(pSelect->pFromTable, walker, pContext);
nodesWalkNode(pSelect->pWhere, walker, pContext);
nodesWalkExpr(pSelect->pFromTable, walker, pContext);
nodesWalkExpr(pSelect->pWhere, walker, pContext);
case SQL_CLAUSE_WHERE:
nodesWalkList(pSelect->pPartitionByList, walker, pContext);
nodesWalkExprs(pSelect->pPartitionByList, walker, pContext);
case SQL_CLAUSE_PARTITION_BY:
nodesWalkNode(pSelect->pWindow, walker, pContext);
nodesWalkExpr(pSelect->pWindow, walker, pContext);
case SQL_CLAUSE_WINDOW:
nodesWalkList(pSelect->pGroupByList, walker, pContext);
nodesWalkExprs(pSelect->pGroupByList, walker, pContext);
case SQL_CLAUSE_GROUP_BY:
nodesWalkNode(pSelect->pHaving, walker, pContext);
nodesWalkExpr(pSelect->pHaving, walker, pContext);
case SQL_CLAUSE_HAVING:
nodesWalkList(pSelect->pOrderByList, walker, pContext);
nodesWalkExprs(pSelect->pOrderByList, walker, pContext);
case SQL_CLAUSE_ORDER_BY:
nodesWalkList(pSelect->pProjectionList, walker, pContext);
nodesWalkExprs(pSelect->pProjectionList, walker, pContext);
case SQL_CLAUSE_SELECT:
default:
break;
@ -319,20 +319,20 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
switch (clause) {
case SQL_CLAUSE_FROM:
nodesRewriteNode(&(pSelect->pFromTable), rewriter, pContext);
nodesRewriteNode(&(pSelect->pWhere), rewriter, pContext);
nodesRewriteExpr(&(pSelect->pFromTable), rewriter, pContext);
nodesRewriteExpr(&(pSelect->pWhere), rewriter, pContext);
case SQL_CLAUSE_WHERE:
nodesRewriteList(pSelect->pPartitionByList, rewriter, pContext);
nodesRewriteExprs(pSelect->pPartitionByList, rewriter, pContext);
case SQL_CLAUSE_PARTITION_BY:
nodesRewriteNode(&(pSelect->pWindow), rewriter, pContext);
nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext);
case SQL_CLAUSE_WINDOW:
nodesRewriteList(pSelect->pGroupByList, rewriter, pContext);
nodesRewriteExprs(pSelect->pGroupByList, rewriter, pContext);
case SQL_CLAUSE_GROUP_BY:
nodesRewriteNode(&(pSelect->pHaving), rewriter, pContext);
nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext);
case SQL_CLAUSE_HAVING:
nodesRewriteList(pSelect->pProjectionList, rewriter, pContext);
nodesRewriteExprs(pSelect->pProjectionList, rewriter, pContext);
case SQL_CLAUSE_SELECT:
nodesRewriteList(pSelect->pOrderByList, rewriter, pContext);
nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext);
default:
break;
}

2
source/libs/nodes/test/nodesTest.cpp
View File

@ -49,7 +49,7 @@ TEST(NodesTest, traverseTest) {
EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR);
EDealRes res = DEAL_RES_CONTINUE;
nodesRewriteNodePostOrder(&pRoot, rewriterTest, &res);
nodesRewriteExprPostOrder(&pRoot, rewriterTest, &res);
EXPECT_EQ(res, DEAL_RES_CONTINUE);
EXPECT_EQ(nodeType(pRoot), QUERY_NODE_VALUE);
EXPECT_EQ(string(((SValueNode*)pRoot)->literal), "18");

2
source/libs/parser/CMakeLists.txt
View File

@ -8,7 +8,7 @@ target_include_directories(
target_link_libraries(
parser
PRIVATE os util nodes catalog function transport qcom
PRIVATE os util nodes catalog function scalar transport qcom
)
if(${BUILD_TEST})

5
source/libs/parser/inc/parInt.h
View File

@ -23,9 +23,10 @@ extern "C" {
#include "parser.h"
int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery);
int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery);
int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery);
int32_t parse(SParseContext* pParseCxt, SQuery** pQuery);
int32_t translate(SParseContext* pParseCxt, SQuery* pQuery);
int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);
int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery);
#ifdef __cplusplus
}

14
source/libs/parser/src/parAstCreater.c
View File

@ -243,13 +243,12 @@ static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptio
return pOptions;
}
char val[20] = {0};
int32_t len = trimString(pVal->z, pVal->n, val, sizeof(val));
char* pStart = val;
char* pEnd = val + len;
if (pVal->n > 2) {
char* pStart = pVal->z + 1;
char* pEnd = pVal->z + pVal->n - 1;
int32_t sepOrder = 1;
while (1) {
char* pPos = strchr(pStart, (0 == sepOrder % 2) ? ',' : ':');
char* pPos = strchr(pStart, (0 == (sepOrder++) % 2) ? ',' : ':');
SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)};
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) {
pCxt->valid = false;
@ -261,9 +260,10 @@ static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptio
}
pStart = pPos + 1;
}
}
if (LIST_LENGTH(pOptions->pRetentions) % 2 != 0) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", val);
if (LIST_LENGTH(pOptions->pRetentions) < 2 || LIST_LENGTH(pOptions->pRetentions) % 2 != 0) {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", pVal->z);
pCxt->valid = false;
}

2
source/libs/parser/src/parAstParser.c
View File

@ -26,7 +26,7 @@ extern void Parse(void*, int, SToken, void*);
extern void ParseFree(void*, FFree);
extern void ParseTrace(FILE*, char*);
int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) {
int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) {
SAstCreateContext cxt;
initAstCreateContext(pParseCxt, &cxt);
void *pParser = ParseAlloc((FMalloc)taosMemoryMalloc);

132
source/libs/parser/src/parCalcConst.c Normal file
View File

@ -0,0 +1,132 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "functionMgt.h"
#include "parInt.h"
#include "scalar.h"
typedef struct SCalcConstContext {
int32_t code;
} SCalcConstContext;
static int32_t calcConstQuery(SNode* pStmt);
static EDealRes doCalcConst(SNode** pNode, SCalcConstContext* pCxt) {
SNode* pNew = NULL;
pCxt->code = scalarCalculateConstants(*pNode, &pNew);
if (TSDB_CODE_SUCCESS != pCxt->code) {
return DEAL_RES_ERROR;
}
((SValueNode*)pNew)->genByCalc = true;
((SValueNode*)pNew)->translate = true;
*pNode = pNew;
return DEAL_RES_CONTINUE;
}
static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) {
SOperatorNode* pOp = *pNode;
if (QUERY_NODE_VALUE == nodeType(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) {
return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext);
}
return DEAL_RES_CONTINUE;
}
static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) {
SFunctionNode* pFunc = *pNode;
if (fmIsPseudoColumnFunc(pFunc->funcId)) {
return DEAL_RES_CONTINUE;
}
SNode* pParam = NULL;
FOREACH(pParam, pFunc->pParameterList) {
if (QUERY_NODE_VALUE != nodeType(pParam)) {
return DEAL_RES_CONTINUE;
}
}
return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext);
}
static EDealRes calcConstLogicCond(SLogicConditionNode** pNode, void* pContext) {
SLogicConditionNode* pCond = *pNode;
SNode* pParam = NULL;
FOREACH(pParam, pCond->pParameterList) {
if (QUERY_NODE_VALUE != nodeType(pParam)) {
return DEAL_RES_CONTINUE;
}
}
return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext);
}
static EDealRes calcConstSubquery(STempTableNode** pNode, void* pContext) {
SCalcConstContext* pCxt = pContext;
pCxt->code = calcConstQuery((*pNode)->pSubquery);
return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_CONTINUE : DEAL_RES_ERROR);
}
static EDealRes calcConst(SNode** pNode, void* pContext) {
switch (nodeType(*pNode)) {
case QUERY_NODE_OPERATOR:
return calcConstOperator((SOperatorNode**)pNode, pContext);
case QUERY_NODE_FUNCTION:
return calcConstFunction((SFunctionNode**)pNode, pContext);
case QUERY_NODE_LOGIC_CONDITION:
return calcConstLogicCond((SLogicConditionNode**)pNode, pContext);
case QUERY_NODE_TEMP_TABLE:
return calcConstSubquery((STempTableNode**)pNode, pContext);
default:
break;
}
return DEAL_RES_CONTINUE;
}
static int32_t calcConstSelect(SSelectStmt* pSelect) {
SCalcConstContext cxt = { .code = TSDB_CODE_SUCCESS };
nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, &cxt);
if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, &cxt);
}
if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprPostOrder(&pSelect->pWhere, calcConst, &cxt);
}
if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt);
}
if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprPostOrder(&pSelect->pWindow, calcConst, &cxt);
}
if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt);
}
if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprPostOrder(&pSelect->pHaving, calcConst, &cxt);
}
if (TSDB_CODE_SUCCESS == cxt.code) {
nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt);
}
return cxt.code;
}
static int32_t calcConstQuery(SNode* pStmt) {
switch (nodeType(pStmt)) {
case QUERY_NODE_SELECT_STMT:
return calcConstSelect((SSelectStmt*)pStmt);
default:
break;
}
return TSDB_CODE_SUCCESS;
}
int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) {
return calcConstQuery(pQuery->pRoot);
}

34
source/libs/parser/src/parTranslater.c
View File

@ -50,14 +50,12 @@ static bool beforeHaving(ESqlClause clause) {
return clause < SQL_CLAUSE_HAVING;
}
static EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t errCode, ...) {
va_list vArgList;
va_start(vArgList, errCode);
generateSyntaxErrMsg(&pCxt->msgBuf, errCode, vArgList);
va_end(vArgList);
pCxt->errCode = errCode;
return DEAL_RES_ERROR;
}
#define generateDealNodeErrMsg(pCxt, code, ...) \
({ \
generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__); \
pCxt->errCode = code; \
DEAL_RES_ERROR; \
})
static int32_t addNamespace(STranslateContext* pCxt, void* pTable) {
size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel);
@ -440,6 +438,9 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
if (nodesIsUnaryOp(pOp)) {
if (OP_TYPE_MINUS == pOp->opType && !IS_NUMERIC_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName);
}
return DEAL_RES_CONTINUE;
}
SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType;
@ -510,12 +511,12 @@ static EDealRes doTranslateExpr(SNode* pNode, void* pContext) {
}
static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) {
nodesWalkNodePostOrder(pNode, doTranslateExpr, pCxt);
nodesWalkExprPostOrder(pNode, doTranslateExpr, pCxt);
return pCxt->errCode;
}
static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) {
nodesWalkListPostOrder(pList, doTranslateExpr, pCxt);
nodesWalkExprsPostOrder(pList, doTranslateExpr, pCxt);
return pCxt->errCode;
}
@ -570,7 +571,7 @@ static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) {
}
static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) {
nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt);
nodesWalkExpr(pNode, doCheckExprForGroupBy, pCxt);
return pCxt->errCode;
}
@ -578,7 +579,7 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList
if (NULL == getGroupByList(pCxt)) {
return TSDB_CODE_SUCCESS;
}
nodesWalkList(pList, doCheckExprForGroupBy, pCxt);
nodesWalkExprs(pList, doCheckExprForGroupBy, pCxt);
return pCxt->errCode;
}
@ -605,9 +606,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect)
return TSDB_CODE_SUCCESS;
}
CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false };
nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt);
nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt);
if (!pSelect->isDistinct) {
nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt);
nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt);
}
if (cxt.existAggFunc && cxt.existCol) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP);
@ -938,7 +939,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe
SNode* pNode = NULL;
int32_t index = 0;
FOREACH(pNode, pRetentions) {
if (0 == index % 2) {
if (0 == ((index++) & 1)) {
pFreq = (SValueNode*)pNode;
} else {
pKeep = (SValueNode*)pNode;
@ -951,6 +952,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe
taosArrayPush(pReq->pRetensions, &retention);
}
}
pReq->numOfRetensions = taosArrayGetSize(pReq->pRetensions);
}
return TSDB_CODE_SUCCESS;
}
@ -2635,7 +2637,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
return TSDB_CODE_SUCCESS;
}
int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) {
int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) {
STranslateContext cxt = {
.pParseCxt = pParseCxt,
.errCode = TSDB_CODE_SUCCESS,

7
source/libs/parser/src/parser.c
View File

@ -30,9 +30,12 @@ static bool isInsertSql(const char* pStr, size_t length) {
}
static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) {
int32_t code = doParse(pCxt, pQuery);
int32_t code = parse(pCxt, pQuery);
if (TSDB_CODE_SUCCESS == code) {
code = doTranslate(pCxt, *pQuery);
code = translate(pCxt, *pQuery);
}
if (TSDB_CODE_SUCCESS == code) {
code = calculateConstant(pCxt, *pQuery);
}
return code;
}

4
source/libs/parser/test/parserAstTest.cpp
View File

@ -54,7 +54,7 @@ private:
static const int max_err_len = 1024;
bool runImpl(int32_t parseCode, int32_t translateCode) {
int32_t code = doParse(&cxt_, &query_);
int32_t code = parse(&cxt_, &query_);
if (code != TSDB_CODE_SUCCESS) {
parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_;
return (terrno == parseCode);
@ -63,7 +63,7 @@ private:
return false;
}
parsedAstStr_ = toString(query_->pRoot);
code = doTranslate(&cxt_, query_);
code = translate(&cxt_, query_);
if (code != TSDB_CODE_SUCCESS) {
translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_;
return (terrno == translateCode);

4
source/libs/planner/src/planLogicCreater.c
View File

@ -83,7 +83,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
}
static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
nodesWalkList(pExprs, doNameExpr, NULL);
nodesWalkExprs(pExprs, doNameExpr, NULL);
SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
return cxt.errCode;
@ -384,7 +384,7 @@ static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pEx
return TSDB_CODE_OUT_OF_MEMORY;
}
nodesWalkList(pExprs, doCreateColumn, &cxt);
nodesWalkExprs(pExprs, doCreateColumn, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(cxt.pList);
return cxt.errCode;

6
source/libs/planner/src/planPhysiCreater.c
View File

@ -264,7 +264,7 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i
.pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId),
.pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))
};
nodesWalkNode(pRes, doSetSlotId, &cxt);
nodesWalkExpr(pRes, doSetSlotId, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyNode(pRes);
return cxt.errCode;
@ -285,7 +285,7 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i
.pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId),
.pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId))
};
nodesWalkList(pRes, doSetSlotId, &cxt);
nodesWalkExprs(pRes, doSetSlotId, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(pRes);
return cxt.errCode;
@ -606,7 +606,7 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN
}
}
SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs };
nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt);
nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt);
if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) {
nodesDestroyList(cxt.pPrecalcExprs);
*pPrecalcExprs = NULL;

7
source/libs/planner/test/plannerTest.cpp
View File

@ -157,6 +157,13 @@ TEST_F(PlannerTest, simple) {
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, selectConstant) {
setDatabase("root", "test");
bind("SELECT 2-1 FROM t1");
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, stSimple) {
setDatabase("root", "test");

2
source/libs/scalar/inc/sclInt.h
View File

@ -47,7 +47,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out);
SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows);
#define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type)
#define GET_PARAM_BYTES(_c) ((_c)->pColumnInfoData->info.bytes)
#define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes)
void sclFreeParam(SScalarParam *param);

8
source/libs/scalar/src/filter.c
View File

@ -1297,7 +1297,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) {
resGroup = taosArrayInit(4, sizeof(SFilterGroup));
SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup};
nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx);
nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)&tctx);
FLT_ERR_JRET(tctx.code);
FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup));
@ -1322,7 +1322,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) {
if (LOGIC_COND_TYPE_OR == node->condType) {
SListCell *cell = node->pParameterList->pHead;
for (int32_t i = 0; i < node->pParameterList->length; ++i) {
nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext);
nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)pContext);
FLT_ERR_JRET(ctx->code);
cell = cell->pNext;
@ -3190,7 +3190,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) {
filterInitUnitsFields(info);
SFltBuildGroupCtx tctx = {.info = info, .group = group};
nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx);
nodesWalkExpr(tree, fltTreeToGroup, (void *)&tctx);
FLT_ERR_JRET(tctx.code);
filterConvertGroupFromArray(info, group);
@ -3566,7 +3566,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
}
int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat);
nodesRewriteExprPostOrder(pNode, fltReviseRewriter, (void *)pStat);
FLT_RET(pStat->code);
}

4
source/libs/scalar/src/scalar.c
View File

@ -670,7 +670,7 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) {
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx);
nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx);
SCL_ERR_JRET(ctx.code);
*pRes = pNode;
@ -694,7 +694,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx);
nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx);
SCL_ERR_JRET(ctx.code);
if (pDst) {

479
source/libs/scalar/src/sclfunc.c
View File

@ -4,7 +4,18 @@
#include "sclInt.h"
#include "sclvector.h"
typedef float (*_float_fn)(float);
typedef double (*_double_fn)(double);
typedef double (*_double_fn_2)(double, double);
typedef int (*_conv_fn)(int);
typedef void (*_trim_fn)(char *, char*, int32_t, int32_t);
typedef int16_t (*_len_fn)(char *, int32_t);
/** Math functions **/
double tlog(double v, double base) {
return log(v) / log(base);
}
int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
SColumnInfoData *pInputData = pInput->columnData;
SColumnInfoData *pOutputData = pOutput->columnData;
@ -102,14 +113,6 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
return TSDB_CODE_SUCCESS;
}
typedef float (*_float_fn)(float);
typedef double (*_double_fn)(double);
typedef double (*_double_fn_2)(double, double);
double tlog(double v, double base) {
return log(v) / log(base);
}
int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn valFn) {
int32_t type = GET_PARAM_TYPE(pInput);
if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) {
@ -211,6 +214,407 @@ int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* p
return TSDB_CODE_SUCCESS;
}
/** String functions **/
int16_t tlength(char *input, int32_t type) {
return varDataLen(input);
}
int16_t tcharlength(char *input, int32_t type) {
if (type == TSDB_DATA_TYPE_VARCHAR) {
return varDataLen(input);
} else { //NCHAR
return varDataLen(input) / TSDB_NCHAR_SIZE;
}
}
void tltrim(char *input, char *output, int32_t type, int32_t charLen) {
int32_t numOfSpaces = 0;
if (type == TSDB_DATA_TYPE_VARCHAR) {
for (int32_t i = 0; i < charLen; ++i) {
if (!isspace(*(varDataVal(input) + i))) {
break;
}
numOfSpaces++;
}
} else { //NCHAR
for (int32_t i = 0; i < charLen; ++i) {
if (!iswspace(*((uint32_t *)varDataVal(input) + i))) {
break;
}
numOfSpaces++;
}
}
int32_t resLen;
if (type == TSDB_DATA_TYPE_VARCHAR) {
resLen = charLen - numOfSpaces;
memcpy(varDataVal(output), varDataVal(input) + numOfSpaces, resLen);
} else {
resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE;
memcpy(varDataVal(output), varDataVal(input) + numOfSpaces * TSDB_NCHAR_SIZE, resLen);
}
varDataSetLen(output, resLen);
}
void trtrim(char *input, char *output, int32_t type, int32_t charLen) {
int32_t numOfSpaces = 0;
if (type == TSDB_DATA_TYPE_VARCHAR) {
for (int32_t i = charLen - 1; i >= 0; --i) {
if (!isspace(*(varDataVal(input) + i))) {
break;
}
numOfSpaces++;
}
} else { //NCHAR
for (int32_t i = charLen - 1; i < charLen; ++i) {
if (!iswspace(*((uint32_t *)varDataVal(input) + i))) {
break;
}
numOfSpaces++;
}
}
int32_t resLen;
if (type == TSDB_DATA_TYPE_VARCHAR) {
resLen = charLen - numOfSpaces;
} else {
resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE;
}
memcpy(varDataVal(output), varDataVal(input), resLen);
varDataSetLen(output, resLen);
}
int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _len_fn lenFn) {
int32_t type = GET_PARAM_TYPE(pInput);
if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) {
return TSDB_CODE_FAILED;
}
SColumnInfoData *pInputData = pInput->columnData;
SColumnInfoData *pOutputData = pOutput->columnData;
char *in = pInputData->pData;
int16_t *out = (int16_t *)pOutputData->pData;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_f(pInputData->nullbitmap, i)) {
colDataSetNull_f(pOutputData->nullbitmap, i);
continue;
}
out[i] = lenFn(in, type);
in += varDataTLen(in);
}
pOutput->numOfRows = pInput->numOfRows;
return TSDB_CODE_SUCCESS;
}
static void setVarTypeOutputBuf(SColumnInfoData *pOutputData, int32_t len, int32_t type) {
pOutputData->pData = taosMemoryCalloc(len, sizeof(char));
pOutputData->info.type = type;
pOutputData->info.bytes = len;
pOutputData->varmeta.length = len;
pOutputData->varmeta.allocLen = len;
}
int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum < 2 || inputNum > 8) { // concat accpet 2-8 input strings
return TSDB_CODE_FAILED;
}
SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *));
SColumnInfoData *pOutputData = pOutput->columnData;
char **input = taosMemoryCalloc(inputNum, POINTER_BYTES);
char *outputBuf = NULL;
int32_t inputLen = 0;
int32_t numOfRows = 0;
for (int32_t i = 0; i < inputNum; ++i) {
if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) ||
GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) {
return TSDB_CODE_FAILED;
}
if (pInput[i].numOfRows > numOfRows) {
numOfRows = pInput[i].numOfRows;
}
}
for (int32_t i = 0; i < inputNum; ++i) {
pInputData[i] = pInput[i].columnData;
input[i] = pInputData[i]->pData;
if (pInput[i].numOfRows == 1) {
inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows;
} else {
inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE;
}
}
int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE;
outputBuf = taosMemoryCalloc(outputLen, 1);
char *output = outputBuf;
bool hasNull = false;
for (int32_t k = 0; k < numOfRows; ++k) {
for (int32_t i = 0; i < inputNum; ++i) {
if (colDataIsNull_s(pInputData[i], k)) {
colDataAppendNULL(pOutputData, k);
hasNull = true;
break;
}
}
if (hasNull) {
continue;
}
int16_t dataLen = 0;
for (int32_t i = 0; i < inputNum; ++i) {
memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i]));
dataLen += varDataLen(input[i]);
if (pInput[i].numOfRows != 1) {
input[i] += varDataTLen(input[i]);
}
}
varDataSetLen(output, dataLen);
colDataAppend(pOutputData, k, output, false);
output += varDataTLen(output);
}
pOutput->numOfRows = numOfRows;
taosMemoryFree(input);
taosMemoryFree(outputBuf);
taosMemoryFree(pInputData);
return TSDB_CODE_SUCCESS;
}
int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum < 3 || inputNum > 9) { // concat accpet 3-9 input strings including the separator
return TSDB_CODE_FAILED;
}
SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *));
SColumnInfoData *pOutputData = pOutput->columnData;
char **input = taosMemoryCalloc(inputNum, POINTER_BYTES);
char *outputBuf = NULL;
int32_t inputLen = 0;
int32_t numOfRows = 0;
for (int32_t i = 1; i < inputNum; ++i) {
if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) ||
GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[1])) {
return TSDB_CODE_FAILED;
}
if (pInput[i].numOfRows > numOfRows) {
numOfRows = pInput[i].numOfRows;
}
}
for (int32_t i = 0; i < inputNum; ++i) {
pInputData[i] = pInput[i].columnData;
if (i == 0) {
// calculate required separator space
int32_t factor = (GET_PARAM_TYPE(&pInput[1]) == TSDB_DATA_TYPE_NCHAR) ? TSDB_NCHAR_SIZE : 1;
inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2) * factor;
} else if (pInput[i].numOfRows == 1) {
inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows;
} else {
inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE;
}
input[i] = pInputData[i]->pData;
}
int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE;
outputBuf = taosMemoryCalloc(outputLen, 1);
char *output = outputBuf;
for (int32_t k = 0; k < numOfRows; ++k) {
if (colDataIsNull_s(pInputData[0], k)) {
colDataAppendNULL(pOutputData, k);
continue;
}
int16_t dataLen = 0;
for (int32_t i = 1; i < inputNum; ++i) {
if (colDataIsNull_s(pInputData[i], k)) {
continue;
}
memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i]));
dataLen += varDataLen(input[i]);
if (pInput[i].numOfRows != 1) {
input[i] += varDataTLen(input[i]);
}
if (i < inputNum - 1) {
//insert the separator
char *sep = pInputData[0]->pData;
memcpy(varDataVal(output) + dataLen, varDataVal(sep), varDataLen(sep));
dataLen += varDataLen(sep);
}
}
varDataSetLen(output, dataLen);
colDataAppend(pOutputData, k, output, false);
output += varDataTLen(output);
}
pOutput->numOfRows = numOfRows;
taosMemoryFree(input);
taosMemoryFree(outputBuf);
taosMemoryFree(pInputData);
return TSDB_CODE_SUCCESS;
}
int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) {
int32_t type = GET_PARAM_TYPE(pInput);
if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) {
return TSDB_CODE_FAILED;
}
SColumnInfoData *pInputData = pInput->columnData;
SColumnInfoData *pOutputData = pOutput->columnData;
char *input = pInputData->pData;
char *output = NULL;
int32_t outputLen = pInputData->varmeta.length;
char *outputBuf = taosMemoryCalloc(outputLen, 1);
output = outputBuf;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
int32_t len = varDataLen(input);
if (type == TSDB_DATA_TYPE_VARCHAR) {
for (int32_t j = 0; j < len; ++j) {
*(varDataVal(output) + j) = convFn(*(varDataVal(input) + j));
}
} else { //NCHAR
for (int32_t j = 0; j < len / TSDB_NCHAR_SIZE; ++j) {
*((uint32_t *)varDataVal(output) + j) = convFn(*((uint32_t *)varDataVal(input) + j));
}
}
varDataSetLen(output, len);
colDataAppend(pOutputData, i, output, false);
input += varDataTLen(input);
output += varDataTLen(output);
}
pOutput->numOfRows = pInput->numOfRows;
taosMemoryFree(outputBuf);
return TSDB_CODE_SUCCESS;
}
int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _trim_fn trimFn) {
int32_t type = GET_PARAM_TYPE(pInput);
if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) {
return TSDB_CODE_FAILED;
}
SColumnInfoData *pInputData = pInput->columnData;
SColumnInfoData *pOutputData = pOutput->columnData;
char *input = pInputData->pData;
char *output = NULL;
int32_t outputLen = pInputData->varmeta.length;
char *outputBuf = taosMemoryCalloc(outputLen, 1);
output = outputBuf;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
int32_t len = varDataLen(input);
int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE;
trimFn(input, output, type, charLen);
varDataSetLen(output, len);
colDataAppend(pOutputData, i, output, false);
input += varDataTLen(input);
output += varDataTLen(output);
}
pOutput->numOfRows = pInput->numOfRows;
taosMemoryFree(outputBuf);
return TSDB_CODE_SUCCESS;
}
int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
if (inputNum != 2 && inputNum!= 3) {
return TSDB_CODE_FAILED;
}
int32_t subPos = 0;
GET_TYPED_DATA(subPos, int32_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
if (subPos == 0) { //subPos needs to be positive or negative values;
return TSDB_CODE_FAILED;
}
int32_t subLen = INT16_MAX;
if (inputNum == 3) {
GET_TYPED_DATA(subLen, int32_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData);
if (subLen < 0) { //subLen cannot be negative
return TSDB_CODE_FAILED;
}
subLen = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subLen : subLen * TSDB_NCHAR_SIZE;
}
SColumnInfoData *pInputData = pInput->columnData;
SColumnInfoData *pOutputData = pOutput->columnData;
char *input = pInputData->pData;
char *output = NULL;
int32_t outputLen = pInputData->varmeta.length;
char *outputBuf = taosMemoryCalloc(outputLen, 1);
output = outputBuf;
for (int32_t i = 0; i < pInput->numOfRows; ++i) {
if (colDataIsNull_s(pInputData, i)) {
colDataAppendNULL(pOutputData, i);
continue;
}
int32_t len = varDataLen(input);
int32_t startPosBytes;
if (subPos > 0) {
startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subPos - 1 : (subPos - 1) * TSDB_NCHAR_SIZE;
startPosBytes = MIN(startPosBytes, len);
} else {
startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? len + subPos : len + subPos * TSDB_NCHAR_SIZE;
startPosBytes = MAX(startPosBytes, 0);
}
subLen = MIN(subLen, len - startPosBytes);
if (subLen > 0) {
memcpy(varDataVal(output), varDataVal(input) + startPosBytes, subLen);
}
varDataSetLen(output, subLen);
colDataAppend(pOutputData, i , output, false);
input += varDataTLen(input);
output += varDataTLen(output);
}
pOutput->numOfRows = pInput->numOfRows;
taosMemoryFree(outputBuf);
return TSDB_CODE_SUCCESS;
}
int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return doScalarFunctionUnique(pInput, inputNum, pOutput, atan);
}
@ -259,57 +663,28 @@ int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
return doScalarFunction(pInput, inputNum, pOutput, roundf, round);
}
static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
assert(numOfInput == 1);
#if 0
int64_t* out = (int64_t*) pOutput->data;
char* s = pLeft->data;
for(int32_t i = 0; i < pLeft->num; ++i) {
out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes));
}
#endif
int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return doCaseConvFunction(pInput, inputNum, pOutput, tolower);
}
static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
assert(numOfInput > 0);
#if 0
int32_t rowLen = 0;
int32_t num = 1;
for(int32_t i = 0; i < numOfInput; ++i) {
rowLen += pLeft[i].bytes;
if (pLeft[i].num > 1) {
num = pLeft[i].num;
}
}
pOutput->data = taosMemoryRealloc(pOutput->data, rowLen * num);
assert(pOutput->data);
char* rstart = pOutput->data;
for(int32_t i = 0; i < num; ++i) {
char* s = rstart;
varDataSetLen(s, 0);
for (int32_t j = 0; j < numOfInput; ++j) {
char* p1 = POINTER_SHIFT(pLeft[j].data, i * pLeft[j].bytes);
memcpy(varDataVal(s) + varDataLen(s), varDataVal(p1), varDataLen(p1));
varDataLen(s) += varDataLen(p1);
}
rstart += rowLen;
}
#endif
int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return doCaseConvFunction(pInput, inputNum, pOutput, toupper);
}
static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return doTrimFunction(pInput, inputNum, pOutput, tltrim);
}
static void trtrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) {
int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return doTrimFunction(pInput, inputNum, pOutput, trtrim);
}
int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return doLengthFunction(pInput, inputNum, pOutput, tlength);
}
int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
return doLengthFunction(pInput, inputNum, pOutput, tcharlength);
}
static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) {

4
source/libs/transport/src/transSrv.c
View File

@ -193,6 +193,7 @@ static void uvHandleReq(SSrvConn* pConn) {
transMsg.ahandle = (void*)pHead->ahandle;
transMsg.handle = NULL;
// transDestroyBuffer(&pConn->readBuf);
transClearBuffer(&pConn->readBuf);
pConn->inType = pHead->msgType;
if (pConn->status == ConnNormal) {
@ -249,6 +250,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
conn->broken = true;
if (conn->status == ConnAcquire) {
if (conn->regArg.init) {
tTrace("server conn %p broken, notify server app", conn);
STrans* pTransInst = conn->pTransInst;
(*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL);
memset(&conn->regArg, 0, sizeof(conn->regArg));
@ -270,7 +272,7 @@ void uvOnTimeoutCb(uv_timer_t* handle) {
void uvOnSendCb(uv_write_t* req, int status) {
SSrvConn* conn = req->data;
transClearBuffer(&conn->readBuf);
// transClearBuffer(&conn->readBuf);
if (status == 0) {
tTrace("server conn %p data already was written on stream", conn);
if (!transQueueEmpty(&conn->srvMsgs)) {

27
source/libs/transport/test/rclient.c
View File

@ -35,8 +35,8 @@ typedef struct {
} SInfo;
static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
SInfo *pInfo = (SInfo *)pMsg->ahandle;
tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
pMsg->code);
// tError("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
// pMsg->code);
if (pEpSet) pInfo->epSet = *pEpSet;
@ -51,9 +51,9 @@ static void *sendRequest(void *param) {
SInfo * pInfo = (SInfo *)param;
SRpcMsg rpcMsg = {0};
tDebug("thread:%d, start to send request", pInfo->index);
tError("thread:%d, start to send request", pInfo->index);
tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs);
tError("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs);
int u100 = 0;
int u500 = 0;
int u1000 = 0;
@ -68,7 +68,7 @@ static void *sendRequest(void *param) {
// tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
int64_t start = taosGetTimestampUs();
rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
if (pInfo->num % 20000 == 0) tError("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
// tsem_wait(&pInfo->rspSem);
tsem_wait(&pInfo->rspSem);
int64_t end = taosGetTimestampUs() - start;
@ -88,7 +88,7 @@ static void *sendRequest(void *param) {
}
tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000);
tDebug("thread:%d, it is over", pInfo->index);
tError("thread:%d, it is over", pInfo->index);
tcount++;
return NULL;
@ -124,7 +124,7 @@ int main(int argc, char *argv[]) {
rpcInit.ckey = "key";
rpcInit.spi = 1;
rpcInit.connType = TAOS_CONN_CLIENT;
rpcDebugFlag = 143;
rpcDebugFlag = 131;
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
@ -170,6 +170,10 @@ int main(int argc, char *argv[]) {
}
}
const char *path = "/tmp/transport/client";
taosRemoveDir(path);
taosMkDir(path);
tstrncpy(tsLogDir, path, PATH_MAX);
taosInitLog("client.log", 10);
void *pRpc = rpcOpen(&rpcInit);
@ -178,8 +182,8 @@ int main(int argc, char *argv[]) {
return -1;
}
tInfo("client is initialized");
tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs);
tError("client is initialized");
tError("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs);
taosGetTimeOfDay(&systemTime);
startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
@ -208,8 +212,9 @@ int main(int argc, char *argv[]) {
endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
float usedTime = (endTime - startTime) / 1000.0f; // mseconds
tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads);
tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize);
tError("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads);
tError("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime,
msgSize);
int ch = getchar();
UNUSED(ch);

7
source/libs/transport/test/rserver.c
View File

@ -125,7 +125,7 @@ int main(int argc, char *argv[]) {
rpcInit.idleTime = 2 * 1500;
rpcInit.afp = retrieveAuthInfo;
rpcDebugFlag = 143;
rpcDebugFlag = 131;
for (int i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-p") == 0 && i < argc - 1) {
@ -160,6 +160,11 @@ int main(int argc, char *argv[]) {
tsAsyncLog = 0;
rpcInit.connType = TAOS_CONN_SERVER;
const char *path = "/tmp/transport/server";
taosRemoveDir(path);
taosMkDir(path);
tstrncpy(tsLogDir, path, PATH_MAX);
taosInitLog("server.log", 10);
void *pRpc = rpcOpen(&rpcInit);

18
source/os/src/osFile.c
View File

@ -325,7 +325,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) {
#if FILE_WITH_LOCK
taosThreadRwlockRdlock(&(pFile->rwlock));
#endif
assert(pFile->fd >= 0);
assert(pFile->fd >= 0); // Please check if you have closed the file.
int64_t leftbytes = count;
int64_t readbytes;
char *tbuf = (char *)buf;
@ -365,7 +365,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset)
#if FILE_WITH_LOCK
taosThreadRwlockRdlock(&(pFile->rwlock));
#endif
assert(pFile->fd >= 0);
assert(pFile->fd >= 0); // Please check if you have closed the file.
int64_t ret = pread(pFile->fd, buf, count, offset);
#if FILE_WITH_LOCK
taosThreadRwlockUnlock(&(pFile->rwlock));
@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
#if FILE_WITH_LOCK
taosThreadRwlockWrlock(&(pFile->rwlock));
#endif
assert(pFile->fd >= 0);
assert(pFile->fd >= 0); // Please check if you have closed the file.
int64_t nleft = count;
int64_t nwritten = 0;
@ -414,7 +414,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
#if FILE_WITH_LOCK
taosThreadRwlockRdlock(&(pFile->rwlock));
#endif
assert(pFile->fd >= 0);
assert(pFile->fd >= 0); // Please check if you have closed the file.
int64_t ret = lseek(pFile->fd, (long)offset, whence);
#if FILE_WITH_LOCK
taosThreadRwlockUnlock(&(pFile->rwlock));
@ -429,7 +429,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
if (pFile == NULL) {
return 0;
}
assert(pFile->fd >= 0);
assert(pFile->fd >= 0); // Please check if you have closed the file.
struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat);
@ -456,7 +456,7 @@ int32_t taosLockFile(TdFilePtr pFile) {
if (pFile == NULL) {
return 0;
}
assert(pFile->fd >= 0);
assert(pFile->fd >= 0); // Please check if you have closed the file.
return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB);
#endif
@ -469,7 +469,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) {
if (pFile == NULL) {
return 0;
}
assert(pFile->fd >= 0);
assert(pFile->fd >= 0); // Please check if you have closed the file.
return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB);
#endif
@ -529,7 +529,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) {
if (pFile == NULL) {
return 0;
}
assert(pFile->fd >= 0);
assert(pFile->fd >= 0); // Please check if you have closed the file.
return ftruncate(pFile->fd, l_size);
#endif
@ -750,7 +750,7 @@ void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) {
if (pFile == NULL) {
return NULL;
}
assert(pFile->fd >= 0);
assert(pFile->fd >= 0); // Please check if you have closed the file.
void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0);
return ptr;

4
source/util/src/tcompression.c
View File

@ -28,11 +28,11 @@
*
* BOOLEAN Compression Algorithm:
* We provide two methods for compress boolean types. Because boolean types in C
* code are char bytes with 0 and 1 values only, only one bit can used to discrimenate
* code are char bytes with 0 and 1 values only, only one bit can used to discriminate
* the values.
* 1. The first method is using only 1 bit to represent the boolean value with 1 for
* true and 0 for false. Then the compression rate is 1/8.
* 2. The second method is using run length encoding (RLE) methods. This methos works
* 2. The second method is using run length encoding (RLE) methods. This method works
* better when there are a lot of consecutive true values or false values.
*
* STRING Compression Algorithm:

9
source/util/src/tlog.c
View File

@ -214,6 +214,7 @@ static void *taosThreadToOpenNewFile(void *param) {
tsLogObj.logHandle->pFile = pFile;
tsLogObj.lines = 0;
tsLogObj.openInProgress = 0;
taosSsleep(3);
taosCloseLogByFd(pOldFile);
uInfo(" new log file:%d is opened", tsLogObj.flag);
@ -347,16 +348,12 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
taosThreadMutexInit(&tsLogObj.logMutex, NULL);
taosUmaskFile(0);
TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE);
tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE);
if (pFile == NULL) {
if (tsLogObj.logHandle->pFile == NULL) {
printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno));
return -1;
}
TdFilePtr pOldFile = tsLogObj.logHandle->pFile;
tsLogObj.logHandle->pFile = pFile;
taosUnLockLogFile(pOldFile);
taosCloseFile(&pOldFile);
taosLockLogFile(tsLogObj.logHandle->pFile);
// only an estimate for number of lines

3
tests/script/jenkins/basic.txt
View File

@ -5,6 +5,7 @@
./test.sh -f tsim/user/basic1.sim
# ---- db
./test.sh -f tsim/db/create_all_options.sim
./test.sh -f tsim/db/alter_option.sim
./test.sh -f tsim/db/basic1.sim
./test.sh -f tsim/db/basic2.sim
@ -40,7 +41,7 @@
# ---- tmq
./test.sh -f tsim/tmq/basic.sim
./test.sh -f tsim/tmq/basic1.sim
#./test.sh -f tsim/tmq/oneTopic.sim
./test.sh -f tsim/tmq/oneTopic.sim
#./test.sh -f tsim/tmq/multiTopic.sim
# --- stable

486
tests/script/tsim/db/create_all_options.sim Normal file
View File

@ -0,0 +1,486 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
sql connect
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
$loop_cnt = 0
check_dnode_ready_1:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> rows: $rows
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
if $data00 != 1 then
return -1
endi
if $data01 != localhost:7100 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready_1
endi
if $data14 != ready then
goto check_dnode_ready_1
endi
if $data24 != ready then
goto check_dnode_ready_1
endi
print ============= create database with all options
#database_option: {
# | BLOCKS value [3~1000, default: 6]
# | CACHE value [default: 16]
# | CACHELAST value [0, 1, 2, 3, default: 0]
# | COMP [0 | 1 | 2, default: 2]
# | DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day]
# | FSYNC value [0 ~ 180000 ms, default: 3000]
# | MAXROWS value [200~10000, default: 4096]
# | MINROWS value [10~1000, default: 100]
# | KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day]
# | PRECISION ['ms' | 'us' | 'ns', default: ms]
# | QUORUM value [1 | 2, default: 1]
# | REPLICA value [1 | 3, default: 1]
# | TTL value [1d ~ , default: 1]
# | WAL value [1 | 2, default: 1]
# | VGROUPS value [default: 2]
# | SINGLE_STABLE [0 | 1, default: ]
# | STREAM_MODE [0 | 1, default: ]
#
#$data0_db : name
#$data1_db : create_time
#$data2_db : vgroups
#$data3_db : ntables
#$data4_db : replica
#$data5_db : quorum
#$data6_db : days
#$data7_db : keep
#$data8_db : cache
#$data9_db : blocks
#$data10_db : minrows
#$data11_db : maxrows
#$data12_db : wal
#$data13_db : fsync
#$data14_db : comp
#$data15_db : cachelast
#$data16_db : precision
print ====> create database db, with default
sql create database db
sql show databases
print rows: $rows
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $rows != 2 then
return -1
endi
if $data0_db != db then # name
return -1
endi
if $data2_db != 2 then # vgroups
return -1
endi
if $data3_db != 0 then # ntables
return -1
endi
if $data4_db != 1 then # replica
return -1
endi
if $data5_db != 1 then # quorum
return -1
endi
if $data6_db != 14400 then # days
return -1
endi
if $data7_db != 5256000,5256000,5256000 then # keep
return -1
endi
if $data8_db != 16 then # cache
return -1
endi
if $data9_db != 6 then # blocks
return -1
endi
if $data10_db != 100 then # minrows
return -1
endi
if $data11_db != 4096 then # maxrows
return -1
endi
if $data12_db != 1 then # wal
return -1
endi
if $data13_db != 3000 then # fsync
return -1
endi
if $data14_db != 2 then # comp
return -1
endi
if $data15_db != 0 then # cachelast
return -1
endi
if $data16_db != ms then # precision
return -1
endi
sql drop database db
print ====> BLOCKS value [3~1000, default: 6]
sql create database db BLOCKS 3
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data9_db != 3 then
return -1
endi
sql drop database db
sql create database db BLOCKS 1000
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data9_db != 1000 then
return -1
endi
sql drop database db
sql_error create database db BLOCKS 2
sql_error create database db BLOCKS 0
sql_error create database db BLOCKS -1
print ====> CACHE value [default: 16]
sql create database db CACHE 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data8_db != 1 then
return -1
endi
sql drop database db
sql create database db CACHE 128
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data8_db != 128 then
return -1
endi
sql drop database db
print ====> CACHELAST value [0, 1, 2, 3, default: 0]
sql create database db CACHELAST 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data15_db != 1 then
return -1
endi
sql drop database db
sql create database db CACHELAST 2
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data15_db != 2 then
return -1
endi
sql drop database db
sql create database db CACHELAST 3
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data15_db != 3 then
return -1
endi
sql drop database db
sql_error create database db CACHELAST 4
sql_error create database db CACHELAST -1
print ====> COMP [0 | 1 | 2, default: 2]
sql create database db COMP 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data14_db != 1 then
return -1
endi
sql drop database db
sql create database db COMP 0
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data14_db != 0 then
return -1
endi
sql drop database db
sql_error create database db COMP 3
sql_error create database db COMP -1
#print ====> DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day]
#print ====> KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day]
#sql create database db DAYS 60m KEEP 60m
#sql show databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data6_db != 60 then
# return -1
#endi
#if $data7_db != 60,60,60 then
# return -1
#endi
#sql drop database db
#sql create database db DAYS 60m KEEP 1d
#sql show databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data6_db != 60 then
# return -1
#endi
#if $data7_db != 1440,1440,1440 then
# return -1
#endi
#sql create database db DAYS 3650d KEEP 365000d
#sql show databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data6_db != 5256000 then
# return -1
#endi
#if $data7_db != 525600000,525600000,525600000 then
# return -1
#endi
#sql drop database db
#sql_error create database db DAYS -59m
#sql_error create database db DAYS 59m
#sql_error create database db DAYS 5256001m
#sql_error create database db DAYS 3651d
#sql_error create database db KEEP -59m
#sql_error create database db KEEP 14399m
#sql_error create database db KEEP 525600001m
#sql_error create database db KEEP 365001d
print ====> FSYNC value [0 ~ 180000 ms, default: 3000]
sql create database db FSYNC 0
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data13_db != 0 then
return -1
endi
sql drop database db
sql create database db FSYNC 180000
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data13_db != 180000 then
return -1
endi
sql drop database db
sql_error create database db FSYNC 180001
sql_error create database db FSYNC -1
print ====> MAXROWS value [200~10000, default: 4096], MINROWS value [10~1000, default: 100]
sql create database db MAXROWS 10000 MINROWS 1000
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data10_db != 1000 then
return -1
endi
if $data11_db != 10000 then
return -1
endi
sql drop database db
sql create database db MAXROWS 200 MINROWS 10
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data10_db != 10 then
return -1
endi
if $data11_db != 200 then
return -1
endi
sql drop database db
sql_error create database db MAXROWS -1
sql_error create database db MAXROWS 0
sql_error create database db MAXROWS 199
sql_error create database db MAXROWS 10001
sql_error create database db MINROWS -1
sql_error create database db MINROWS 0
sql_error create database db MINROWS 9
sql_error create database db MINROWS 1001
sql_error create database db MAXROWS 500 MINROWS 1000
print ====> PRECISION ['ms' | 'us' | 'ns', default: ms]
sql create database db PRECISION 'us'
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data16_db != us then
return -1
endi
sql drop database db
sql create database db PRECISION 'ns'
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data16_db != ns then
return -1
endi
sql drop database db
sql_error create database db PRECISION 'as'
sql_error create database db PRECISION -1
print ====> QUORUM value [1 | 2, default: 1]
#sql create database db QUORUM 2
#sql show databases
#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $data5_db != 2 then
# return -1
#endi
#sql drop database db
sql create database db QUORUM 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data5_db != 1 then
return -1
endi
sql drop database db
sql_error create database db QUORUM 3
sql_error create database db QUORUM 0
sql_error create database db QUORUM -1
print ====> REPLICA value [1 | 3, default: 1]
sql create database db REPLICA 3
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data4_db != 3 then
return -1
endi
sql drop database db
sql create database db REPLICA 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data4_db != 1 then
return -1
endi
sql drop database db
sql_error create database db REPLICA 2
sql_error create database db REPLICA 0
sql_error create database db REPLICA -1
sql_error create database db REPLICA 4
print ====> TTL value [1d ~ , default: 1]
sql create database db TTL 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXX_db != 1 then
# return -1
#endi
sql drop database db
sql create database db TTL 10
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXX_db != 10 then
# return -1
#endi
sql drop database db
sql_error create database db TTL 0
sql_error create database db TTL -1
print ====> WAL value [1 | 2, default: 1]
sql create database db WAL 2
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data12_db != 2 then
return -1
endi
sql drop database db
sql create database db WAL 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data12_db != 1 then
return -1
endi
sql drop database db
sql_error create database db WAL 3
sql_error create database db WAL -1
sql_error create database db WAL 0
print ====> VGROUPS value [1~4096, default: 2]
sql create database db VGROUPS 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data2_db != 1 then
return -1
endi
sql drop database db
sql create database db VGROUPS 16
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
if $data2_db != 16 then
return -1
endi
sql drop database db
sql_error create database db VGROUPS 4097
sql_error create database db VGROUPS -1
sql_error create database db VGROUPS 0
print ====> SINGLE_STABLE [0 | 1, default: ]
sql create database db SINGLE_STABLE 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXXXX_db != 1 then
# return -1
#endi
sql drop database db
sql create database db SINGLE_STABLE 0
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXXXX_db != 0 then
# return -1
#endi
sql drop database db
sql_error create database db SINGLE_STABLE 2
sql_error create database db SINGLE_STABLE -1
print ====> STREAM_MODE [0 | 1, default: ]
sql create database db STREAM_MODE 1
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXXX_db != 1 then
# return -1
#endi
sql drop database db
sql create database db STREAM_MODE 0
sql show databases
print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db
#if $dataXXX_db != 0 then
# return -1
#endi
sql drop database db
sql_error create database db STREAM_MODE 2
sql_error create database db STREAM_MODE -1
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

148
tests/script/tsim/query/session.sim
View File

@ -79,7 +79,7 @@ loop_test_pos:
# session(ts,5a)
print ====> select count(*) from dev_001 session(ts,5a)
sql select count(*) from dev_001 session(ts,5a)
sql select _wstartts, count(*) from dev_001 session(ts,5a)
print ====> rows: $rows
print ====> $data00 $data01 $data02 $data03 $data04 $data05
print ====> $data10 $data11 $data12 $data13 $data14 $data15
@ -100,7 +100,7 @@ endi
print ====> select count(*) from (select * from dev_001) session(ts,5a)
sql select count(*) from (select * from dev_001) session(ts,5a)
sql select _wstartts, count(*) from (select * from dev_001) session(ts,5a)
if $rows != 15 then
return -1
endi
@ -109,7 +109,7 @@ if $data01 != 2 then
endi
print ====> select count(*) from dev_001 session(ts,1s)
sql select count(*) from dev_001 session(ts,1s)
sql select _wstartts, count(*) from dev_001 session(ts,1s)
if $rows != 12 then
return -1
endi
@ -118,7 +118,7 @@ if $data01 != 5 then
endi
print ====> select count(*) from (select * from dev_001) session(ts,1s)
sql select count(*) from (select * from dev_001) session(ts,1s)
sql select _wstartts, count(*) from (select * from dev_001) session(ts,1s)
if $rows != 12 then
return -1
endi
@ -127,7 +127,7 @@ if $data01 != 5 then
endi
print ====> select count(*) from dev_001 session(ts,1000a)
sql select count(*) from dev_001 session(ts,1000a)
sql select _wstartts, count(*) from dev_001 session(ts,1000a)
if $rows != 12 then
return -1
endi
@ -136,7 +136,7 @@ if $data01 != 5 then
endi
print ====> select count(*) from (select * from dev_001) session(ts,1000a)
sql select count(*) from (select * from dev_001) session(ts,1000a)
sql select _wstartts, count(*) from (select * from dev_001) session(ts,1000a)
if $rows != 12 then
return -1
endi
@ -145,7 +145,7 @@ if $data01 != 5 then
endi
print ====> select count(*) from dev_001 session(ts,1m)
sql select count(*) from dev_001 session(ts,1m)
sql select _wstartts, count(*) from dev_001 session(ts,1m)
if $rows != 9 then
return -1
endi
@ -154,7 +154,7 @@ if $data01 != 8 then
endi
print ====> select count(*) from (select * from dev_001) session(ts,1m)
sql select count(*) from (select * from dev_001) session(ts,1m)
sql select _wstartts, count(*) from (select * from dev_001) session(ts,1m)
if $rows != 9 then
return -1
endi
@ -163,7 +163,7 @@ if $data01 != 8 then
endi
print ====> select count(*) from dev_001 session(ts,1h)
sql select count(*) from dev_001 session(ts,1h)
sql select _wstartts, count(*) from dev_001 session(ts,1h)
if $rows != 6 then
return -1
endi
@ -172,7 +172,7 @@ if $data01 != 11 then
endi
print ====> select count(*) from (select * from dev_001) session(ts,1h)
sql select count(*) from (select * from dev_001) session(ts,1h)
sql select _wstartts, count(*) from (select * from dev_001) session(ts,1h)
if $rows != 6 then
return -1
endi
@ -181,7 +181,7 @@ if $data01 != 11 then
endi
print ====> select count(*) from dev_001 session(ts,1d)
sql select count(*) from dev_001 session(ts,1d)
sql select _wstartts, count(*) from dev_001 session(ts,1d)
if $rows != 4 then
return -1
endi
@ -190,7 +190,7 @@ if $data01 != 13 then
endi
print ====> select count(*) from (select * from dev_001) session(ts,1d)
sql select count(*) from (select * from dev_001) session(ts,1d)
sql select _wstartts, count(*) from (select * from dev_001) session(ts,1d)
if $rows != 4 then
return -1
endi
@ -199,7 +199,7 @@ if $data01 != 13 then
endi
print ====> select count(*) from dev_001 session(ts,1w)
sql select count(*) from dev_001 session(ts,1w)
sql select _wstartts, count(*) from dev_001 session(ts,1w)
if $rows != 2 then
return -1
endi
@ -208,7 +208,7 @@ if $data01 != 15 then
endi
print ====> select count(*) from (select * from dev_001) session(ts,1w)
sql select count(*) from (select * from dev_001) session(ts,1w)
sql select _wstartts, count(*) from (select * from dev_001) session(ts,1w)
if $rows != 2 then
return -1
endi
@ -216,71 +216,73 @@ if $data01 != 15 then
return -1
endi
print ====> leastsquares not supported yet.
print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)
sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)
if $rows != 2 then
return -1
endi
if $data01 != 13 then
return -1
endi
if $data02 != 1 then
return -1
endi
if $data03 != 13 then
return -1
endi
if $data04 != 7 then
return -1
endi
if $data05 != 91 then
return -1
endi
if $data06 != 1 then
return -1
endi
if $data07 != 13 then
return -1
endi
if $data08 != @{slop:1.000000, intercept:0.000000}@ then
return -1
endi
if $data09 != 12 then
return -1
endi
#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d)
#if $rows != 2 then
# return -1
#endi
#if $data01 != 13 then
# return -1
#endi
#if $data02 != 1 then
# return -1
#endi
#if $data03 != 13 then
# return -1
#endi
#if $data04 != 7 then
# return -1
#endi
#if $data05 != 91 then
# return -1
#endi
#if $data06 != 1 then
# return -1
#endi
#if $data07 != 13 then
# return -1
#endi
#if $data08 != @{slop:1.000000, intercept:0.000000}@ then
# return -1
#endi
#if $data09 != 12 then
# return -1
#endi
# $data0-10 != 3.741657387
# $data0-11 != 1
# $data1-11 != 14
print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)
sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)
if $rows != 2 then
return -1
endi
if $data01 != 13 then
return -1
endi
if $data02 != 1 then
return -1
endi
if $data03 != 13 then
return -1
endi
if $data04 != 7 then
return -1
endi
if $data05 != 91 then
return -1
endi
if $data06 != 1 then
return -1
endi
if $data07 != 13 then
return -1
endi
if $data08 != @{slop:1.000000, intercept:0.000000}@ then
return -1
endi
#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d)
#if $rows != 2 then
# return -1
#endi
#if $data01 != 13 then
# return -1
#endi
#if $data02 != 1 then
# return -1
#endi
#if $data03 != 13 then
# return -1
#endi
#if $data04 != 7 then
# return -1
#endi
#if $data05 != 91 then
# return -1
#endi
#if $data06 != 1 then
# return -1
#endi
#if $data07 != 13 then
# return -1
#endi
#if $data08 != @{slop:1.000000, intercept:0.000000}@ then
# return -1
#endi
sql_error select * from dev_001 session(ts,1w)
sql_error select count(*) from st session(ts,1w)