diff --git a/cmake/cmake.define b/cmake/cmake.define index fb6ba8cc2e..851d8f9a73 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -44,7 +44,6 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) SET(COMMON_FLAGS "/W3 /D_WIN32") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) diff --git a/example/src/tmq.c b/example/src/tmq.c index abd4f78610..976d658fa6 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -14,9 +14,7 @@ */ #include -#include #include -#include #include #include #include "taos.h" diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3d6bb06015..ea605090f9 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -842,7 +842,7 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int64_t dbUid; int32_t vgVersion; - int32_t numOfStables; + int32_t numOfStables; int32_t buffer; int32_t pageSize; int32_t pages; @@ -1442,32 +1442,32 @@ typedef struct { SArray* lostConsumers; // SArray SArray* removedConsumers; // SArray SArray* newConsumers; // SArray -} SMqRebSubscribe; +} SMqRebInfo; -static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { - SMqRebSubscribe* pRebSub = (SMqRebSubscribe*)taosMemoryCalloc(1, sizeof(SMqRebSubscribe)); - if (pRebSub == NULL) { +static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { + SMqRebInfo* pRebInfo = (SMqRebInfo*)taosMemoryCalloc(1, sizeof(SMqRebInfo)); + if (pRebInfo == NULL) { goto _err; } - strcpy(pRebSub->key, key); - pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t)); - if (pRebSub->lostConsumers == NULL) { + strcpy(pRebInfo->key, key); + pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); + if (pRebInfo->lostConsumers == NULL) { goto _err; } - pRebSub->removedConsumers = taosArrayInit(0, sizeof(int64_t)); - if (pRebSub->removedConsumers == NULL) { + pRebInfo->removedConsumers = taosArrayInit(0, sizeof(int64_t)); + if (pRebInfo->removedConsumers == NULL) { goto _err; } - pRebSub->newConsumers = taosArrayInit(0, sizeof(int64_t)); - if (pRebSub->newConsumers == NULL) { + pRebInfo->newConsumers = taosArrayInit(0, sizeof(int64_t)); + if (pRebInfo->newConsumers == NULL) { goto _err; } - return pRebSub; + return pRebInfo; _err: - taosArrayDestroy(pRebSub->lostConsumers); - taosArrayDestroy(pRebSub->removedConsumers); - taosArrayDestroy(pRebSub->newConsumers); - taosMemoryFreeClear(pRebSub); + taosArrayDestroy(pRebInfo->lostConsumers); + taosArrayDestroy(pRebInfo->removedConsumers); + taosArrayDestroy(pRebInfo->newConsumers); + taosMemoryFreeClear(pRebInfo); return NULL; } diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 0332b8ce69..5f1ee157e1 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -231,6 +231,7 @@ typedef struct SSelectStmt { uint8_t precision; bool isEmptyResult; bool hasAggFuncs; + bool isTimeOrderQuery; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; diff --git a/include/os/osDef.h b/include/os/osDef.h index 8da0c2c00c..6f6199de7a 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -62,7 +62,7 @@ extern "C" { #define strncasecmp _strnicmp #define wcsncasecmp _wcsnicmp #define strtok_r strtok_s - #define snprintf _snprintf + // #define snprintf _snprintf #define in_addr_t unsigned long // #define socklen_t int diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 118fbf1a03..263fd0bad5 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -517,6 +517,7 @@ void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp); typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; SRWLatch lock; + int64_t dbUid; int32_t vgNum; int8_t subType; int8_t withTbName; @@ -553,9 +554,8 @@ int32_t tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogO void* tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog); typedef struct { - const SMqSubscribeObj* pOldSub; - const SMqTopicObj* pTopic; - const SMqRebSubscribe* pRebInfo; + int32_t oldConsumerNum; + const SMqRebInfo* pRebInfo; } SMqRebInputObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index fd82c60d37..e3174a90a2 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -31,6 +31,8 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic); SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw); +int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index b17772bdb2..23a87b4691 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -134,15 +134,15 @@ FAIL: return -1; } -static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); +static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { + SMqRebInfo *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); if (pRebSub == NULL) { pRebSub = tNewSMqRebSubscribe(key); if (pRebSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe)); + taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebInfo)); } return pRebSub; } @@ -189,7 +189,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { char key[TSDB_SUBSCRIBE_KEY_LEN]; char *removedTopic = taosArrayGetP(pConsumer->currentTopics, i); mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } taosRUnLockLatch(&pConsumer->lock); @@ -200,7 +200,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { char key[TSDB_SUBSCRIBE_KEY_LEN]; char *newTopic = taosArrayGetP(pConsumer->rebNewTopics, i); mndMakeSubscribeKey(key, pConsumer->cgroup, newTopic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); } @@ -209,7 +209,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { char key[TSDB_SUBSCRIBE_KEY_LEN]; char *removedTopic = taosArrayGetP(pConsumer->rebRemovedTopics, i); mndMakeSubscribeKey(key, pConsumer->cgroup, removedTopic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); + SMqRebInfo *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } taosRUnLockLatch(&pConsumer->lock); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 03ac74aa62..5bce6ac218 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -241,7 +241,7 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) { void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp) { buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); - buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp)); + buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp)); #if 0 int32_t sz; buf = taosDecodeFixedI32(buf, &sz); @@ -277,6 +277,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); taosInitRWLatch(&pSubNew->lock); + pSubNew->dbUid = pSub->dbUid; pSubNew->subType = pSub->subType; pSubNew->withTbName = pSub->withTbName; pSubNew->withSchema = pSub->withSchema; @@ -310,6 +311,7 @@ void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { int32_t tlen = 0; tlen += taosEncodeString(buf, pSub->key); + tlen += taosEncodeFixedI64(buf, pSub->dbUid); tlen += taosEncodeFixedI32(buf, pSub->vgNum); tlen += taosEncodeFixedI8(buf, pSub->subType); tlen += taosEncodeFixedI8(buf, pSub->withTbName); @@ -336,6 +338,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { // buf = taosDecodeStringTo(buf, pSub->key); + buf = taosDecodeFixedI64(buf, &pSub->dbUid); buf = taosDecodeFixedI32(buf, &pSub->vgNum); buf = taosDecodeFixedI8(buf, &pSub->subType); buf = taosDecodeFixedI8(buf, &pSub->withTbName); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 1349439dd2..8ae0d5d19c 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -731,7 +731,6 @@ _OVER: static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; int32_t code = -1; - SStbObj *pTopicStb = NULL; SStbObj *pStb = NULL; SDbObj *pDb = NULL; SUserObj *pUser = NULL; @@ -762,12 +761,6 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { goto _OVER; } - pTopicStb = mndAcquireStb(pMnode, createReq.name); - if (pTopicStb != NULL) { - terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC; - goto _OVER; - } - pDb = mndAcquireDbByStb(pMnode, createReq.name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -785,7 +778,7 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) { int32_t numOfStbs = -1; mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs); - if (pDb->cfg.numOfStables == 1 && numOfStbs != 0 ) { + if (pDb->cfg.numOfStables == 1 && numOfStbs != 0) { terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB; goto _OVER; } @@ -799,7 +792,6 @@ _OVER: } mndReleaseStb(pMnode, pStb); - mndReleaseStb(pMnode, pTopicStb); mndReleaseDb(pMnode, pDb); mndReleaseUser(pMnode, pUser); tFreeSMCreateStbReq(&createReq); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 13ee26b6cd..545caea03d 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -80,6 +80,7 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pSub->dbUid = pTopic->dbUid; pSub->subType = pTopic->subType; pSub->withTbName = pTopic->withTbName; pSub->withSchema = pTopic->withSchema; @@ -174,27 +175,20 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) return 0; } -static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); +static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { + SMqRebInfo *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); if (pRebSub == NULL) { pRebSub = tNewSMqRebSubscribe(key); if (pRebSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe)); + taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebInfo)); } return pRebSub; } static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { - if (pInput->pTopic != NULL) { - // create subscribe - pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key); - ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 0); - } else { - pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub); - } int32_t totalVgNum = pOutput->pSub->vgNum; mInfo("mq rebalance subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum); @@ -245,12 +239,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } // 3. calc vg number of each consumer - int32_t oldSz = 0; - if (pInput->pOldSub) { - oldSz = taosHashGetSize(pInput->pOldSub->consumerHash); - } - int32_t afterRebConsumerNum = - oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers); + int32_t afterRebConsumerNum = pInput->oldConsumerNum + taosArrayGetSize(pInput->pRebInfo->newConsumers) - + taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t minVgCnt = 0; int32_t imbConsumerNum = 0; // calc num @@ -488,22 +478,34 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { rebOutput.touchedConsumers = taosArrayInit(0, sizeof(void *)); rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg)); - SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; - SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); + SMqRebInfo *pRebInfo = (SMqRebInfo *)pIter; + SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebInfo->key); + + rebInput.pRebInfo = pRebInfo; if (pSub == NULL) { // split sub key and extract topic char topic[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CGROUP_LEN]; - mndSplitSubscribeKey(pRebSub->key, topic, cgroup); + mndSplitSubscribeKey(pRebInfo->key, topic, cgroup); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); ASSERT(pTopic); taosRLockLatch(&pTopic->lock); - rebInput.pTopic = pTopic; - } - rebInput.pRebInfo = pRebSub; - rebInput.pOldSub = pSub; + rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key); + ASSERT(taosHashGetSize(rebOutput.pSub->consumerHash) == 0); + + taosRUnLockLatch(&pTopic->lock); + mndReleaseTopic(pMnode, pTopic); + + rebInput.oldConsumerNum = 0; + } else { + taosRLockLatch(&pSub->lock); + rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash); + rebOutput.pSub = tCloneSubscribeObj(pSub); + taosRUnLockLatch(&pSub->lock); + mndReleaseSubscribe(pMnode, pSub); + } // TODO replace assert with error check ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); @@ -516,14 +518,6 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { mError("persist rebalance output error, possibly vnode splitted or dropped"); } - - if (rebInput.pTopic) { - SMqTopicObj *pTopic = (SMqTopicObj *)rebInput.pTopic; - taosRUnLockLatch(&pTopic->lock); - mndReleaseTopic(pMnode, pTopic); - } else { - mndReleaseSubscribe(pMnode, pSub); - } } // reset flag @@ -593,7 +587,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); - buf = taosMemoryMalloc(tlen + 1); + buf = taosMemoryMalloc(tlen); if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); @@ -679,3 +673,36 @@ static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } + +static int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { + SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqSubscribeObj *pSub = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub); + if (pIter == NULL) break; + + if (pSub->dbUid != pDb->uid) { + sdbRelease(pSdb, pSub); + continue; + } + + if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) { + goto END; + } + } + + code = 0; +END: + return code; +} diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 731ee69105..0a8d1cee4a 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -38,6 +38,8 @@ static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp); static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); +static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); + int32_t mndInitTopic(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_TOPIC, .keyType = SDB_KEY_BINARY, @@ -553,7 +555,41 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB return numOfRows; } +static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic) { + SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqTopicObj *pTopic = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; + + if (pTopic->dbUid != pDb->uid) { + sdbRelease(pSdb, pTopic); + continue; + } + + if (mndSetDropTopicCommitLogs(pMnode, pTrans, pTopic) < 0) { + goto END; + } + } + + code = 0; +END: + return code; +} diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a39d959b36..411d5c451c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -58,6 +58,48 @@ void tqClose(STQ* pTq) { // TODO } +static void tdSRowDemo() { +#define DEMO_N_COLS 3 + + int16_t schemaVersion = 0; + int32_t numOfCols = DEMO_N_COLS; // ts + int + SRowBuilder rb = {0}; + + SSchema schema[DEMO_N_COLS] = { + {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = SCHEMA_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = SCHEMA_SMA_ON}, + {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = SCHEMA_SMA_ON}}; + + SSchema* pSchema = schema; + STSchema* pTSChema = tdGetSTSChemaFromSSChema(&pSchema, numOfCols); + + tdSRowInit(&rb, schemaVersion); + tdSRowSetTpInfo(&rb, numOfCols, pTSChema->flen); + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSChema); + void* row = taosMemoryCalloc(1, maxLen); // make sure the buffer is enough + + // set row buf + tdSRowResetBuf(&rb, row); + + for (int32_t idx = 0; idx < pTSChema->numOfCols; ++idx) { + STColumn* pColumn = pTSChema->columns + idx; + if (idx == 0) { + int64_t tsKey = 1651234567; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &tsKey, true, pColumn->offset, idx); + } else if (idx == 1) { + int32_t val1 = 10; + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &val1, true, pColumn->offset, idx); + } else { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, true, pColumn->offset, idx); + } + } + + // print + tdSRowPrint(row, pTSChema, __func__); + + taosMemoryFree(pTSChema); +} + int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { if (msgType != TDMT_VND_SUBMIT) return 0; void* pIter = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ed8a978226..4eeba06027 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -45,47 +45,6 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { #endif return 0; } -static void tdSRowDemo() { -#define DEMO_N_COLS 3 - - int16_t schemaVersion = 0; - int32_t numOfCols = DEMO_N_COLS; // ts + int - SRowBuilder rb = {0}; - - SSchema schema[DEMO_N_COLS] = { - {.type = TSDB_DATA_TYPE_TIMESTAMP, .colId = 1, .name = "ts", .bytes = 8, .flags = SCHEMA_SMA_ON}, - {.type = TSDB_DATA_TYPE_INT, .colId = 2, .name = "c1", .bytes = 4, .flags = SCHEMA_SMA_ON}, - {.type = TSDB_DATA_TYPE_INT, .colId = 3, .name = "c2", .bytes = 4, .flags = SCHEMA_SMA_ON}}; - - SSchema *pSchema = schema; - STSchema *pTSChema = tdGetSTSChemaFromSSChema(&pSchema, numOfCols); - - tdSRowInit(&rb, schemaVersion); - tdSRowSetTpInfo(&rb, numOfCols, pTSChema->flen); - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSChema); - void *row = taosMemoryCalloc(1, maxLen); // make sure the buffer is enough - - // set row buf - tdSRowResetBuf(&rb, row); - - for (int32_t idx = 0; idx < pTSChema->numOfCols; ++idx) { - STColumn *pColumn = pTSChema->columns + idx; - if (idx == 0) { - int64_t tsKey = 1651234567; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &tsKey, true, pColumn->offset, idx); - } else if (idx == 1) { - int32_t val1 = 10; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, &val1, true, pColumn->offset, idx); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, true, pColumn->offset, idx); - } - } - - // print - tdSRowPrint(row, pTSChema, __func__); - - taosMemoryFree(pTSChema); -} int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { void *ptr = NULL; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1e0b9b3130..8f3c88900d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -221,7 +221,7 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE }; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; return TSDB_CODE_SUCCESS; } @@ -256,8 +256,7 @@ static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = - (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -439,6 +438,7 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +// clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "count", @@ -568,7 +568,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "first", .type = FUNCTION_TYPE_FIRST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -578,7 +578,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last", .type = FUNCTION_TYPE_LAST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -588,7 +588,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "diff", .type = FUNCTION_TYPE_DIFF, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateInOutNum, .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, @@ -976,5 +976,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = NULL } }; +// clang-format on const int32_t funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFuncDefinition)); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 0113da94eb..b505f2e8ec 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -21,11 +21,8 @@ #include "taos.h" #include "taoserror.h" #include "thash.h" -#include "builtins.h" -#include "catalog.h" #include "tudf.h" - typedef struct SFuncMgtService { SHashObj* pFuncNameHashTable; } SFuncMgtService; @@ -148,6 +145,8 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } + bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index df46999379..9aac2bfe0c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1176,7 +1176,7 @@ int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInte // input: interBuf // output: resultData int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { - int8_t callType = TSDB_UDF_CALL_AGG_PROC; + int8_t callType = TSDB_UDF_CALL_AGG_FIN; int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData); return err; } @@ -1243,12 +1243,12 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } SUdfUvSession *session = (SUdfUvSession *)handle; SUdfAggRes *udfRes = (SUdfAggRes*)GET_ROWCELL_INTERBUF(pResultCellInfo); - udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; memset(udfRes, 0, envSize); + udfRes->finalResBuf = (char*)udfRes + sizeof(SUdfAggRes); + udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; + udfRes->session = (SUdfUvSession *)handle; SUdfInterBuf buf = {0}; if (callUdfAggInit(handle, &buf) != 0) { @@ -1260,7 +1260,6 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResult } int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { - SInputColumnInfoData* pInput = &pCtx->input; int32_t numOfCols = pInput->numOfInputCols; @@ -1320,13 +1319,15 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { udfRes->interResBuf = (char*)udfRes + sizeof(SUdfAggRes) + session->outputLen; - SUdfInterBuf resultBuf = {.buf = udfRes->finalResBuf, - .bufLen = session->outputLen, - .numOfResult = udfRes->finalResNum}; + SUdfInterBuf resultBuf = {0}; SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, .numOfResult = udfRes->interResNum}; callUdfAggFinalize(session, &state, &resultBuf); + + udfRes->finalResBuf = resultBuf.buf; + udfRes->finalResNum = resultBuf.numOfResult; + teardownUdf(session); if (resultBuf.numOfResult == 1) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index ae24a832c8..06fa49e1c2 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -124,7 +124,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { char *finishSuffix = "_finish"; strncpy(finishFuncName, processFuncName, strlen(processFuncName)); strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); - uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggFinishFunc)); + uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); //TODO: merge } return 0; diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index 83187c5855..69ed515d2b 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -27,7 +27,7 @@ int32_t udf2_start(SUdfInterBuf *buf) { int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { int64_t sumSquares = *(int64_t*)interBuf->buf; for (int32_t i = 0; i < block->numOfCols; ++i) { - for (int32_t j = 0; j < block->numOfRows; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { SUdfColumn* col = block->udfCols[i]; //TODO: check the bitmap for null value int32_t* rows = (int32_t*)col->colData.fixLenCol.data; @@ -35,7 +35,7 @@ int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInte } } - *(int64_t*)newInterBuf = sumSquares; + *(int64_t*)(newInterBuf->buf) = sumSquares; newInterBuf->bufLen = sizeof(int64_t); //TODO: if all null value, numOfResult = 0; newInterBuf->numOfResult = 1; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index f1fe68c55a..92da476319 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -154,6 +154,7 @@ static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicCond } static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { + COPY_ALL_SCALAR_FIELDS; exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_ARRAY_FIELD(functionName); COPY_SCALAR_FIELD(funcId); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index fa862df2b1..6dcd2a0baf 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2077,6 +2077,7 @@ static const char* jkFunctionName = "Name"; static const char* jkFunctionId = "Id"; static const char* jkFunctionType = "Type"; static const char* jkFunctionParameter = "Parameters"; +static const char* jkFunctionUdfBufSize = "UdfBufSize"; static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { const SFunctionNode* pNode = (const SFunctionNode*)pObj; @@ -2094,6 +2095,9 @@ static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkFunctionParameter, pNode->pParameterList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkFunctionUdfBufSize, pNode->udfBufSize); + } return code; } @@ -2114,6 +2118,9 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkFunctionParameter, &pNode->pParameterList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkFunctionUdfBufSize, &pNode->udfBufSize); + } return code; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 20e1bba77f..1701976f8d 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -586,6 +586,7 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr select->pProjectionList = pProjectionList; select->pFromTable = pTable; sprintf(select->stmtName, "%p", select); + select->isTimeOrderQuery = true; return (SNode*)select; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 63ad7c76ef..2d08342214 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -319,7 +319,7 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol pCol->node.resType = pExpr->resType; } -static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { +static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; int32_t nums = @@ -643,6 +643,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING); } pCxt->pCurrStmt->hasAggFuncs = true; + pCxt->pCurrStmt->isTimeOrderQuery = false; } return DEAL_RES_CONTINUE; @@ -939,7 +940,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) { size_t nums = taosArrayGetSize(pTables); for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); - int32_t code = createColumnNodeByTable(pCxt, pTable, *pCols); + int32_t code = createColumnsByTable(pCxt, pTable, *pCols); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -1011,7 +1012,7 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SN } } if (TSDB_CODE_SUCCESS == code) { - code = createColumnNodeByTable(pCxt, pTable, *pOutput); + code = createColumnsByTable(pCxt, pTable, *pOutput); } return code; } @@ -1093,52 +1094,33 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod return code; } -static bool isCountStar(SNode* pNode) { - if (QUERY_NODE_FUNCTION != nodeType(pNode) || 1 != LIST_LENGTH(((SFunctionNode*)pNode)->pParameterList)) { - return false; - } - SNode* pPara = nodesListGetNode(((SFunctionNode*)pNode)->pParameterList, 0); - return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); -} - -static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { - SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); - if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); - } - return code; -} - static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pProjectionList) { // select * ... return createAllColumns(pCxt, &pSelect->pProjectionList); } else { SNode* pNode = NULL; WHERE_EACH(pNode, pSelect->pProjectionList) { + int32_t code = TSDB_CODE_SUCCESS; if (isMultiResFunc(pNode)) { SNodeList* pFuncs = NULL; - if (TSDB_CODE_SUCCESS != createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs); + if (TSDB_CODE_SUCCESS == code) { + INSERT_LIST(pSelect->pProjectionList, pFuncs); + ERASE_NODE(pSelect->pProjectionList); + continue; } - INSERT_LIST(pSelect->pProjectionList, pFuncs); - ERASE_NODE(pSelect->pProjectionList); - continue; } else if (isTableStar(pNode)) { SNodeList* pCols = NULL; - if (TSDB_CODE_SUCCESS != createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - INSERT_LIST(pSelect->pProjectionList, pCols); - ERASE_NODE(pSelect->pProjectionList); - continue; - } else if (isCountStar(pNode)) { - int32_t code = rewriteCountStar(pCxt, (SFunctionNode*)pNode); - if (TSDB_CODE_SUCCESS != code) { - return code; + code = createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols); + if (TSDB_CODE_SUCCESS == code) { + INSERT_LIST(pSelect->pProjectionList, pCols); + ERASE_NODE(pSelect->pProjectionList); + continue; } } + if (TSDB_CODE_SUCCESS != code) { + return code; + } WHERE_NEXT; } } @@ -1254,12 +1236,14 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL != pSelect->pGroupByList && NULL != pSelect->pWindow) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST); } - pCxt->currClause = SQL_CLAUSE_GROUP_BY; - return translateExprList(pCxt, pSelect->pGroupByList); + if (NULL != pSelect->pGroupByList) { + pCxt->currClause = SQL_CLAUSE_GROUP_BY; + pSelect->isTimeOrderQuery = false; + return translateExprList(pCxt, pSelect->pGroupByList); + } + return TSDB_CODE_SUCCESS; } -static bool isValTimeUnit(char unit) { return ('n' == unit || 'y' == unit); } - static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) { int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd'); switch (unit) { @@ -1286,7 +1270,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = isValTimeUnit(pInter->unit); + bool valInter = TIME_IS_VAR_DURATION(pInter->unit); if (pInter->datum.i <= 0 || (!valInter && convertTimePrecision(pInter->datum.i, precision, TSDB_TIME_PRECISION_MICRO) < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime); @@ -1300,7 +1284,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->unit == 'n' && pOffset->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !isValTimeUnit(pOffset->unit) && !valInter; + bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { @@ -1312,7 +1296,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* const static int32_t INTERVAL_SLIDING_FACTOR = 100; SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (pInter->unit == 'n' || pInter->unit == 'y') { + if (TIME_IS_VAR_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || @@ -1419,6 +1403,78 @@ static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } +static bool isCountStar(SFunctionNode* pFunc) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return false; + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); +} + +// count(*) is rewritten as count(ts) for scannning optimization +static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { + SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } + return code; +} + +static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* pTable, SNode** pPrimaryKey) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } else { + // todo + } + *pPrimaryKey = (SNode*)pCol; + return TSDB_CODE_SUCCESS; +} + +static int32_t createPrimaryKeyCol(STranslateContext* pCxt, SNode** pPrimaryKey) { + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, NULL, &pTable); + if (TSDB_CODE_SUCCESS == code) { + code = createPrimaryKeyColByTable(pCxt, pTable, pPrimaryKey); + } + return code; +} + +static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + SNode* pPrimaryKey = NULL; + int32_t code = createPrimaryKeyCol(pCxt, &pPrimaryKey); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey); + } + return code; +} + +EDealRes rewriteFuncForSelectImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + STranslateContext* pCxt = pContext; + SFunctionNode* pFunc = (SFunctionNode*)pNode; + if (isCountStar(pFunc)) { + pCxt->errCode = rewriteCountStar(pCxt, pFunc); + } else if (fmIsTimelineFunc(pFunc->funcId)) { + pCxt->errCode = rewriteTimelineFunc(pCxt, pFunc); + } + if (TSDB_CODE_SUCCESS != pCxt->errCode) { + return DEAL_RES_ERROR; + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteFuncForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteFuncForSelectImpl, pCxt); + return pCxt->errCode; +} + static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect); @@ -1449,6 +1505,9 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = checkLimit(pCxt, pSelect); } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteFuncForSelect(pCxt, pSelect); + } return code; } @@ -1703,13 +1762,17 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); } - SNode* pNode = NULL; - FOREACH(pNode, pRetentions) { - SNode* pVal = NULL; - FOREACH(pVal, ((SNodeListNode*)pNode)->pNodeList) { - if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pVal)) { + SNode* pRetention = NULL; + FOREACH(pRetention, pRetentions) { + SNode* pNode = NULL; + FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) { + SValueNode* pVal = (SValueNode*)pNode; + if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } + if (!TIME_IS_VAR_DURATION(pVal->unit)) { + pVal->datum.i = convertTimeFromPrecisionToUnit(pVal->datum.i, pVal->node.resType.precision, pVal->unit); + } } } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 990c272612..4c3a4e8ab9 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -80,6 +80,18 @@ TEST_F(ParserSelectTest, multiResFunc) { run("select last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); } +TEST_F(ParserSelectTest, timelineFunc) { + useDb("root", "test"); + + run("select last(*), first(*) from t1"); + + run("select last(*), first(*) from t1 group by c1"); + + run("select last(*), first(*) from t1 interval(10s)"); + + run("select diff(c1) from t1"); +} + TEST_F(ParserSelectTest, clause) { useDb("root", "test"); diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 26b1fb8ece..05a009d79d 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -23,9 +23,9 @@ class PlanBasicTest : public PlannerTestBase {}; TEST_F(PlanBasicTest, select) { useDb("root", "test"); - // run("select * from t1"); - // run("select 1 from t1"); - // run("select * from st1"); + run("select * from t1"); + run("select 1 from t1"); + run("select * from st1"); run("select 1 from st1"); } @@ -40,4 +40,10 @@ TEST_F(PlanBasicTest, join) { run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); run("select t1.c1, t2.c2 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts"); -} \ No newline at end of file +} + +TEST_F(PlanBasicTest, func) { + useDb("root", "test"); + + run("select diff(c1) from t1"); +} diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 73fcffd729..05ffe41fe7 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -42,3 +42,11 @@ TEST_F(PlanGroupByTest, withOrderBy) { // order by alias of aggfunc // run("select count(*), sum(c1) a from t1 order by a"); } + +TEST_F(PlanGroupByTest, aggFunc) { + useDb("root", "test"); + + run("select last(*), first(*) from t1"); + + run("select last(*), first(*) from t1 group by c1"); +} diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index a2b1353093..ca471262ff 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "osDef.h" #include "qworker.h" #include "tlockfree.h" #include "ttimer.h" @@ -301,9 +302,9 @@ typedef struct SQWorkerMgmt { extern SQWorkerMgmt gQwMgmt; -FORCE_INLINE SQWorker *qwAcquire(int64_t refId) { return (SQWorker *)taosAcquireRef(atomic_load_32(&gQwMgmt.qwRef), refId); } +static FORCE_INLINE SQWorker *qwAcquire(int64_t refId) { return (SQWorker *)taosAcquireRef(atomic_load_32(&gQwMgmt.qwRef), refId); } -FORCE_INLINE int32_t qwRelease(int64_t refId) { return taosReleaseRef(gQwMgmt.qwRef, refId); } +static FORCE_INLINE int32_t qwRelease(int64_t refId) { return taosReleaseRef(gQwMgmt.qwRef, refId); } #ifdef __cplusplus diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index b06c774ed3..cf3cd162aa 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -28,6 +28,7 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* if (buf == NULL) { return -1; } + if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { ((SMsgHead*)buf)->vgId = 0; req.taskId = pTask->inplaceDispatcher.taskId;