From e92bfa558dc4c391217c0a3a14b5b34dcaf6e3e1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 6 Mar 2023 21:16:30 +0800 Subject: [PATCH 001/176] fix(query): allow only one trans to be execute for each balance, and do some other refactor. --- source/dnode/mnode/impl/src/mndConsumer.c | 133 ++++++++------- source/dnode/mnode/impl/src/mndSubscribe.c | 186 +++++++++++++-------- 2 files changed, 188 insertions(+), 131 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index f1ef83aca5..623b375305 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -845,6 +845,53 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { return 0; } +static void updateConsumerStatus(SMqConsumerObj* pConsumer) { + int32_t status = pConsumer->status; + + if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { + if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pConsumer->status = MQ_CONSUMER_STATUS__READY; + } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) { + pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; + } + } else { + if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + pConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) { + pConsumer->status = MQ_CONSUMER_STATUS__LOST; + } + } +} + +// remove from new topic +static void removeFromNewTopicList(SMqConsumerObj* pConsumer, const char* pTopic) { + int32_t size = taosArrayGetSize(pConsumer->rebNewTopics); + for (int32_t i = 0; i < taosArrayGetSize(pConsumer->rebNewTopics); i++) { + char *p = taosArrayGetP(pConsumer->rebNewTopics, i); + if (strcmp(pTopic, p) == 0) { + taosArrayRemove(pConsumer->rebNewTopics, i); + taosMemoryFree(p); + + mDebug("consumer:0x%" PRIx64 " remove new topic:%s in the topic list, remain newTopics:%d", pConsumer->consumerId, + pTopic, (int) taosArrayGetSize(pConsumer->rebNewTopics)); + break; + } + } +} + +// remove from removed topic +static void removeFromRemoveTopicList(SMqConsumerObj* pConsumer, const char* pTopic) { + int32_t size = taosArrayGetSize(pConsumer->rebRemovedTopics); + for (int32_t i = 0; i < size; i++) { + char *p = taosArrayGetP(pConsumer->rebRemovedTopics, i); + if (strcmp(pTopic, p) == 0) { + taosArrayRemove(pConsumer->rebRemovedTopics, i); + taosMemoryFree(p); + break; + } + } +} + static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", uptime:%" PRId64, pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime); @@ -855,6 +902,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/ + // this new consumer has identical topics with one existed consumers. if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { pOldConsumer->status = MQ_CONSUMER_STATUS__READY; } else { @@ -871,7 +919,6 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pNewConsumer->assignedTopics = tmp; pOldConsumer->subscribeTime = pNewConsumer->upTime; - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; } } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { @@ -911,71 +958,48 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->rebalanceTime = pNewConsumer->upTime; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { - /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);*/ - /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);*/ + ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); + char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); - char *addedTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); // not exist in current topic - - bool existing = false; -#if 1 + bool existing = false; int32_t numOfExistedTopics = taosArrayGetSize(pOldConsumer->currentTopics); for (int32_t i = 0; i < numOfExistedTopics; i++) { char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); - if (strcmp(topic, addedTopic) == 0) { + if (strcmp(topic, pNewTopic) == 0) { existing = true; } } -#endif - // remove from new topic - for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { - char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); - if (strcmp(addedTopic, topic) == 0) { - taosArrayRemove(pOldConsumer->rebNewTopics, i); - taosMemoryFree(topic); - break; - } - } + removeFromNewTopicList(pOldConsumer, pNewTopic); // add to current topic if (!existing) { - taosArrayPush(pOldConsumer->currentTopics, &addedTopic); + taosArrayPush(pOldConsumer->currentTopics, &pNewTopic); taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString); } else { - taosMemoryFree(addedTopic); + taosMemoryFree(pNewTopic); } // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__READY; - } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) { - pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; - } - } else { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; - } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; - } - } + updateConsumerStatus(pOldConsumer); // the re-balance is triggered when the new consumer is launched. pOldConsumer->rebalanceTime = pNewConsumer->upTime; atomic_add_fetch_32(&pOldConsumer->epoch, 1); - mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d", + mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 + ", current topics:%d, newTopics:%d, removeTopics:%d", pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, - mndConsumerStatusName(pOldConsumer->status), - pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics)); + mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, + (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), + (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/ /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/ char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); - - // not exist in new topic #if 0 for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); @@ -984,14 +1008,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, #endif // remove from removed topic - for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) { - char *topic = taosArrayGetP(pOldConsumer->rebRemovedTopics, i); - if (strcmp(removedTopic, topic) == 0) { - taosArrayRemove(pOldConsumer->rebRemovedTopics, i); - taosMemoryFree(topic); - break; - } - } + removeFromRemoveTopicList(pOldConsumer, removedTopic); // remove from current topic int32_t i = 0; @@ -1004,32 +1021,20 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, break; } } - // must find the topic - /*A(i < sz);*/ // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__READY; - } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) { - pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; - } - } else { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; - } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; - } - } + updateConsumerStatus(pOldConsumer); pOldConsumer->rebalanceTime = pNewConsumer->upTime; atomic_add_fetch_32(&pOldConsumer->epoch, 1); - mDebug("consumer:0x%" PRIx64 " state %d(%s) -> %d(%s), new epoch:%d, reb-time:%" PRId64 ", current topics:%d", + mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 + ", current topics:%d, newTopics:%d, removeTopics:%d", pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, - mndConsumerStatusName(pOldConsumer->status), - pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics)); + mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, + (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), + (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } taosWUnLockLatch(&pOldConsumer->lock); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 21539a6313..5ad12a6450 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -197,24 +197,20 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebSub; } -static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { - int32_t totalVgNum = pOutput->pSub->vgNum; - const char *sub = pOutput->pSub->key; - mInfo("sub:%s mq re-balance %d vgroups", sub, pOutput->pSub->vgNum); +static void doRemoveExistedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { + int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); + const char *pSubKey = pOutput->pSub->key; - // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg - SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - - // 2. check and get actual removed consumers, put their vg into hash - int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t actualRemoved = 0; - for (int32_t i = 0; i < removedNum; i++) { + for (int32_t i = 0; i < numOfRemoved; i++) { uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); + // consumer exists till now if (pConsumerEp) { actualRemoved++; + int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); for (int32_t j = 0; j < consumerVgNum; j++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); @@ -223,52 +219,66 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .newConsumerId = -1, .pVgEp = pVgEp, }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, sub, pVgEp->vgId, consumerId); + mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, pSubKey, pVgEp->vgId, consumerId); } + taosArrayDestroy(pConsumerEp->vgs); taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); + // put into removed taosArrayPush(pOutput->removedConsumers, &consumerId); } } - if (removedNum != actualRemoved) { - mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved); + if (numOfRemoved != actualRemoved) { + mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", pSubKey, numOfRemoved, actualRemoved); + } else { + mInfo("sub:%s removed %d consumers", pSubKey, numOfRemoved); } +} - // if previously no consumer, there are vgs not assigned - { - int32_t consumerVgNum = taosArrayGetSize(pOutput->pSub->unassignedVgs); - for (int32_t i = 0; i < consumerVgNum; i++) { - SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs); - SMqRebOutputVg rebOutput = { - .oldConsumerId = -1, - .newConsumerId = -1, - .pVgEp = pVgEp, - }; - taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", sub, pVgEp->vgId); - } +static void doAddNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) { + int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers); + const char *pSubKey = pOutput->pSub->key; + + for (int32_t i = 0; i < numOfNewConsumers; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); + + SMqConsumerEp newConsumerEp; + newConsumerEp.consumerId = consumerId; + newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); + + taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); + taosArrayPush(pOutput->newConsumers, &consumerId); + mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, pSubKey, consumerId); } +} - // 3. calc vg number of each consumer - int32_t afterRebConsumerNum = pInput->oldConsumerNum + taosArrayGetSize(pInput->pRebInfo->newConsumers) - - taosArrayGetSize(pInput->pRebInfo->removedConsumers); - int32_t minVgCnt = 0; - int32_t imbConsumerNum = 0; - // calc num - if (afterRebConsumerNum) { - minVgCnt = totalVgNum / afterRebConsumerNum; - imbConsumerNum = totalVgNum % afterRebConsumerNum; +static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj* pHash) { + const char *pSubKey = pOutput->pSub->key; + int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs); + + for (int32_t i = 0; i < numOfVgroups; i++) { + SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs); + SMqRebOutputVg rebOutput = { + .oldConsumerId = -1, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); + mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", pSubKey, pVgEp->vgId); } +} - mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has more vgs", sub, - afterRebConsumerNum, minVgCnt, imbConsumerNum); +static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj* pHash, int32_t minVgCnt, int32_t imbConsumerNum) { + const char *pSubKey = pOutput->pSub->key; - // 4. first scan: remove consumer more than wanted, put to remove hash int32_t imbCnt = 0; void *pIter = NULL; + while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) { @@ -276,8 +286,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); - int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); // all old consumers still existing are touched // TODO optimize: touch only consumer whose vgs changed taosArrayPush(pOutput->touchedConsumers, &pConsumerEp->consumerId); @@ -296,13 +306,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId, + mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId, pConsumerEp->consumerId); } imbCnt++; } } else { - // pop until equal minVg + // all the remain consumers should only have the number of vgroups, which is equalled to the value of minVg while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt) { SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs); SMqRebOutputVg outputVg = { @@ -311,36 +321,66 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId, + mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId, pConsumerEp->consumerId); } } } } +} - // 5. add new consumer into sub - { - int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers); - for (int32_t i = 0; i < consumerNum; i++) { - int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); +static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { + int32_t totalVgNum = pOutput->pSub->vgNum; + const char *pSubKey = pOutput->pSub->key; - SMqConsumerEp newConsumerEp; - newConsumerEp.consumerId = consumerId; - newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); - taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); - taosArrayPush(pOutput->newConsumers, &consumerId); - mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, sub, consumerId); - } + int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); + int32_t numOfAdded = taosArrayGetSize(pInput->pRebInfo->newConsumers); + mInfo("sub:%s mq re-balance %d vgroups, existed consumers:%d, added:%d, removed:%d", pSubKey, totalVgNum, + pInput->oldConsumerNum, numOfAdded, numOfRemoved); + + // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg + SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + + // 2. check and get actual removed consumers, put their vg into hash + doRemoveExistedConsumers(pOutput, pHash, pInput); + + // 3. if previously no consumer, there are vgs not assigned + addUnassignedVgroups(pOutput, pHash); + + // 4. calc vg number of each consumer + int32_t numOfFinal = pInput->oldConsumerNum + numOfAdded - numOfRemoved; + + int32_t minVgCnt = 0; + int32_t imbConsumerNum = 0; + + // calc num + if (numOfFinal) { + minVgCnt = totalVgNum / numOfFinal; + imbConsumerNum = totalVgNum % numOfFinal; } - // 6. second scan: find consumer do not have enough vg, extract from temporary hash and assign to new consumer. + mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has 1 more vgroups than avg value", + pSubKey, numOfFinal, minVgCnt, imbConsumerNum); + + // 5. first scan: remove vgroups from te consumers, who have more vgroups than the threashold value that is + // minVgCnt, and then put them into the recycled hash list + transferVgroupsForConsumers(pOutput, pHash, minVgCnt, imbConsumerNum); + + // 6. add new consumer into sub + doAddNewConsumers(pOutput, pInput); + + // 7. second scan: find consumer do not have enough vgroups, extract from temporary hash and assign to them // All related vg should be put into rebVgs SMqRebOutputVg *pRebVg = NULL; void *pRemovedIter = NULL; - pIter = NULL; + void *pIter = NULL; + while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; // push until equal minVg @@ -348,8 +388,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR // iter hash and find one vg pRemovedIter = taosHashIterate(pHash, pRemovedIter); if (pRemovedIter == NULL) { - mError("sub:%s removed iter is null", sub); - continue; + mError("sub:%s removed iter is null", pSubKey); + break; } pRebVg = (SMqRebOutputVg *)pRemovedIter; @@ -409,15 +449,15 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->rebVgs, pRebOutput); - mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId); + mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", pSubKey, pRebOutput->pVgEp->vgId); } } // 8. generate logs - mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", sub); + mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", pSubKey); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, sub, + mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, pSubKey, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } { @@ -427,10 +467,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", sub, pConsumerEp->consumerId, sz); + mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", pSubKey, pConsumerEp->consumerId, sz); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); - mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, sub, pVgEp->vgId, + mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, pSubKey, pVgEp->vgId, pConsumerEp->consumerId); } } @@ -555,17 +595,24 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqDoRebalanceMsg *pReq = pMsg->pCont; void *pIter = NULL; + bool rebalanceExec = false; // to ensure only once. - mInfo("mq re-balance start"); + mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash)); + // here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction. while (1) { + if (rebalanceExec) { + break; + } + pIter = taosHashIterate(pReq->rebSubHash, pIter); if (pIter == NULL) { break; } - SMqRebInputObj rebInput = {0}; + taosSsleep(20); + SMqRebInputObj rebInput = {0}; SMqRebOutputObj rebOutput = {0}; rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t)); rebOutput.removedConsumers = taosArrayInit(0, sizeof(int64_t)); @@ -582,9 +629,10 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { char topic[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CGROUP_LEN]; mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); if (pTopic == NULL) { - mError("mq re-balance %s ignored since topic %s not exist", pRebInfo->key, topic); + mError("mq re-balance %s ignored since topic %s doesn't exist", pRebInfo->key, topic); continue; } @@ -604,11 +652,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { mndReleaseTopic(pMnode, pTopic); rebInput.oldConsumerNum = 0; + mInfo("topic:%s has no consumers sub yet", topic); } else { taosRLockLatch(&pSub->lock); rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash); rebOutput.pSub = tCloneSubscribeObj(pSub); taosRUnLockLatch(&pSub->lock); + mInfo("topic:%s has %d consumers sub till now", pRebInfo->key, rebInput.oldConsumerNum); mndReleaseSubscribe(pMnode, pSub); } @@ -633,6 +683,8 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { taosArrayDestroy(rebOutput.rebVgs); tDeleteSubscribeObj(rebOutput.pSub); taosMemoryFree(rebOutput.pSub); + + rebalanceExec = true; } // reset flag From 1b93f980e874476c0b35f5ba5461659d233924db Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 6 Mar 2023 21:33:31 +0800 Subject: [PATCH 002/176] refactor: do some internal refactor. --- source/dnode/mnode/impl/src/mndSubscribe.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 5ad12a6450..6368472dc9 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -357,11 +357,12 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR if (numOfFinal) { minVgCnt = totalVgNum / numOfFinal; imbConsumerNum = totalVgNum % numOfFinal; + mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has 1 more vgroups than avg value", + pSubKey, numOfFinal, minVgCnt, imbConsumerNum); + } else { + mInfo("sub:%s no consumer subscribe this topic", pSubKey); } - mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has 1 more vgroups than avg value", - pSubKey, numOfFinal, minVgCnt, imbConsumerNum); - // 5. first scan: remove vgroups from te consumers, who have more vgroups than the threashold value that is // minVgCnt, and then put them into the recycled hash list transferVgroupsForConsumers(pOutput, pHash, minVgCnt, imbConsumerNum); From 60cdbe37d39ede2d8aace45b32b158446907ef8f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 6 Mar 2023 22:25:34 +0800 Subject: [PATCH 003/176] refactor: do some internal refactor. --- source/client/test/clientTests.cpp | 5 +++-- source/dnode/mnode/impl/src/mndSubscribe.c | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 2f3d600019..cee818ccf7 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -925,7 +925,8 @@ TEST(clientCase, subscription_test) { // 创建订阅 topics 列表 tmq_list_t* topicList = tmq_list_new(); -// tmq_list_append(topicList, "topic_t1"); + tmq_list_append(topicList, "topic_t1"); + tmq_list_append(topicList, "topic_s2"); // 启动订阅 tmq_subscribe(tmq, topicList); @@ -954,7 +955,7 @@ TEST(clientCase, subscription_test) { printf("db: %s\n", dbName); printf("vgroup id: %d\n", vgroupId); - if (count ++ > 20) { + if (count ++ > 40) { tmq_unsubscribe(tmq); break; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 6368472dc9..5cc2bedfdb 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -611,8 +611,6 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { break; } - taosSsleep(20); - SMqRebInputObj rebInput = {0}; SMqRebOutputObj rebOutput = {0}; rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t)); From fc39a330b4405ce61b971437d63068665e547a28 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 6 Mar 2023 23:30:25 +0800 Subject: [PATCH 004/176] fix:wrong uasge of taosArrayGet --- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 5cc2bedfdb..402f157366 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -823,7 +823,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { if (pSub->unassignedVgs != NULL) { int32_t size = (int32_t)taosArrayGetSize(pSub->unassignedVgs); for (int32_t i = 0; i < size; ++i) { - SMqVgEp *pMqVgEp = taosArrayGet(pSub->unassignedVgs, i); + SMqVgEp *pMqVgEp = (SMqVgEp *)taosArrayGetP(pSub->unassignedVgs, i); tmsgUpdateDnodeEpSet(&pMqVgEp->epSet); } } @@ -833,7 +833,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { SMqConsumerEp *pConsumerEp = pIter; int32_t size = (int32_t)taosArrayGetSize(pConsumerEp->vgs); for (int32_t i = 0; i < size; ++i) { - SMqVgEp *pMqVgEp = taosArrayGet(pConsumerEp->vgs, i); + SMqVgEp *pMqVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); tmsgUpdateDnodeEpSet(&pMqVgEp->epSet); } pIter = taosHashIterate(pSub->consumerHash, pIter); From e350f71396b91ff1905ea666c03fe9e450499a3b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 14 Mar 2023 19:45:31 +0800 Subject: [PATCH 005/176] enh: reset elect timer at the end of callbacks --- source/libs/sync/src/syncAppendEntries.c | 6 +++--- source/libs/sync/src/syncElection.c | 1 - source/libs/sync/src/syncMain.c | 22 ++++++++-------------- source/libs/sync/src/syncRequestVote.c | 5 ++++- source/libs/sync/src/syncSnapshot.c | 2 +- 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index bd1dae54d9..9ab545075c 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -105,6 +105,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SRpcMsg rpcRsp = {0}; bool accepted = false; SSyncRaftEntry* pEntry = NULL; + bool resetElect = false; // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { @@ -137,7 +138,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } syncNodeStepDown(ths, pMsg->term); - syncNodeResetElectTimer(ths); + resetElect = true; if (pMsg->dataLen < sizeof(SSyncRaftEntry)) { sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d", @@ -184,10 +185,9 @@ _SEND_RESPONSE: // commit index, i.e. leader notice me if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { sError("vgId:%d, failed to commit raft fsm log since %s.", ths->vgId, terrstr()); - goto _out; } -_out: + if (resetElect) syncNodeResetElectTimer(ths); return 0; _IGNORE: diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index e53b8ade1c..3699efbc59 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -115,6 +115,5 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { ASSERT(ret == 0); syncNodeResetElectTimer(pSyncNode); - return ret; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index e5fe4a2369..ed97c8abde 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1593,8 +1593,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; syncNodeStopHeartbeatTimer(pSyncNode); - // reset elect timer - syncNodeResetElectTimer(pSyncNode); + // trace log + sNTrace(pSyncNode, "become follower %s", debugStr); // send rsp to client syncNodeLeaderChangeRsp(pSyncNode); @@ -1610,8 +1610,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // reset log buffer syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode); - // trace log - sNTrace(pSyncNode, "become follower %s", debugStr); + // reset elect timer + syncNodeResetElectTimer(pSyncNode); } // TLA+ Spec @@ -2277,6 +2277,7 @@ static int32_t syncNodeAppendNoopOld(SSyncNode* ths) { int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeat* pMsg = pRpcMsg->pCont; + bool resetElect = false; const STraceId* trace = &pRpcMsg->info.traceId; char tbuf[40] = {0}; @@ -2300,12 +2301,11 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { if (pMsg->term == currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs); + resetElect = true; - syncNodeResetElectTimer(ths); ths->minMatchIndex = pMsg->minMatchIndex; if (ths->state == TAOS_SYNC_STATE_FOLLOWER) { - // syncNodeFollowerCommit(ths, pMsg->commitIndex); SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); @@ -2328,7 +2328,6 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } if (pMsg->term >= currentTerm && ths->state != TAOS_SYNC_STATE_FOLLOWER) { - // syncNodeStepDown(ths, pMsg->term); SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); @@ -2348,15 +2347,10 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } } - /* - // htonl - SMsgHead* pHead = rpcMsg.pCont; - pHead->contLen = htonl(pHead->contLen); - pHead->vgId = htonl(pHead->vgId); - */ - // reply syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); + + if (resetElect) syncNodeResetElectTimer(ths); return 0; } diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 2fda2a19b8..40b747b711 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -89,6 +89,7 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int32_t ret = 0; SyncRequestVote* pMsg = pRpcMsg->pCont; + bool resetElect = false; // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &pMsg->srcId)) { @@ -115,7 +116,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncNodeStepDown(ths, currentTerm); // forbid elect for this round - syncNodeResetElectTimer(ths); + resetElect = true; } // send msg @@ -134,5 +135,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncLogRecvRequestVote(ths, pMsg, pReply->voteGranted, ""); syncLogSendRequestVoteReply(ths, pReply, ""); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + + if (resetElect) syncNodeResetElectTimer(ths); return 0; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 413d6adf05..a9d0ddad17 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -798,7 +798,6 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { if (pMsg->term > raftStoreGetTerm(pSyncNode)) { syncNodeStepDown(pSyncNode, pMsg->term); } - syncNodeResetElectTimer(pSyncNode); // state, term, seq/ack int32_t code = 0; @@ -840,6 +839,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { code = -1; } + syncNodeResetElectTimer(pSyncNode); return code; } From cc76d78673e50c04db8613001763bbb8bba2c369 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 14 Mar 2023 20:35:39 +0800 Subject: [PATCH 006/176] enh: separate sync read and write events --- include/common/tmsgcb.h | 2 +- source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 4 ++-- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 17 +++++++++-------- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 16 ++++++++-------- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 4 ++-- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 16 ++++++++-------- source/dnode/mgmt/mgmt_vnode/src/vmInt.c | 6 +++--- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 20 +++++++++----------- source/dnode/mnode/impl/src/mndSync.c | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 10 files changed, 44 insertions(+), 45 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index eca8740d28..9b709272b2 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -34,7 +34,7 @@ typedef enum { WRITE_QUEUE, APPLY_QUEUE, SYNC_QUEUE, - SYNC_CTRL_QUEUE, + SYNC_RD_QUEUE, STREAM_QUEUE, QUEUE_MAX, } EQueueType; diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index b47742b4ed..24f75e3c8b 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -34,7 +34,7 @@ typedef struct SMnodeMgmt { SSingleWorker readWorker; SSingleWorker writeWorker; SSingleWorker syncWorker; - SSingleWorker syncCtrlWorker; + SSingleWorker syncRdWorker; bool stopped; int32_t refCount; TdThreadRwlock lock; @@ -54,7 +54,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt); void mmStopWorker(SMnodeMgmt *pMgmt); int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutMsgToSyncCtrlQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToSyncRdQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index fe65c3dde9..c129ea21ac 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -188,21 +188,22 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DISABLE_WRITE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index b0810d528f..0152e5d0b1 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -111,8 +111,8 @@ int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); } -int32_t mmPutMsgToSyncCtrlQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutMsgToWorker(pMgmt, &pMgmt->syncCtrlWorker, pMsg); +int32_t mmPutMsgToSyncRdQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->syncRdWorker, pMsg); } int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { @@ -151,8 +151,8 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { case SYNC_QUEUE: pWorker = &pMgmt->syncWorker; break; - case SYNC_CTRL_QUEUE: - pWorker = &pMgmt->syncCtrlWorker; + case SYNC_RD_QUEUE: + pWorker = &pMgmt->syncRdWorker; break; default: terrno = TSDB_CODE_INVALID_PARA; @@ -238,12 +238,12 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { SSingleWorkerCfg scCfg = { .min = 1, .max = 1, - .name = "mnode-sync-ctrl", + .name = "mnode-sync-rd", .fp = (FItem)mmProcessSyncMsg, .param = pMgmt, }; - if (tSingleWorkerInit(&pMgmt->syncCtrlWorker, &scCfg) != 0) { - dError("failed to start mnode mnode-sync-ctrl worker since %s", terrstr()); + if (tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg) != 0) { + dError("failed to start mnode mnode-sync-rd worker since %s", terrstr()); return -1; } @@ -259,6 +259,6 @@ void mmStopWorker(SMnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->readWorker); tSingleWorkerCleanup(&pMgmt->writeWorker); tSingleWorkerCleanup(&pMgmt->syncWorker); - tSingleWorkerCleanup(&pMgmt->syncCtrlWorker); + tSingleWorkerCleanup(&pMgmt->syncRdWorker); dDebug("mnode workers are closed"); } diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index e3fa2964b7..4374ae363c 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -59,7 +59,7 @@ typedef struct { SVnode *pImpl; SMultiWorker pWriteW; SMultiWorker pSyncW; - SMultiWorker pSyncCtrlW; + SMultiWorker pSyncRdW; SMultiWorker pApplyW; STaosQueue *pQueryQ; STaosQueue *pStreamQ; @@ -107,7 +107,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); int32_t vmPutMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutMsgToSyncRdQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index d7f91b74a8..b60d2c3caf 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -549,22 +549,22 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 0524e2713a..4c5b1246e7 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -98,9 +98,9 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) pVnode->pSyncW.queue->threadId); tMultiWorkerCleanup(&pVnode->pSyncW); - dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId, - pVnode->pSyncCtrlW.queue, pVnode->pSyncCtrlW.queue->threadId); - tMultiWorkerCleanup(&pVnode->pSyncCtrlW); + dInfo("vgId:%d, wait for vnode sync rd queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue, + pVnode->pSyncRdW.queue->threadId); + tMultiWorkerCleanup(&pVnode->pSyncRdW); dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, pVnode->pApplyW.queue->threadId); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 7aa1c9f56a..e4e0d608de 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -216,9 +216,9 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pSyncW.queue, pMsg); break; - case SYNC_CTRL_QUEUE: - dGTrace("vgId:%d, msg:%p put into vnode-sync-ctrl queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pSyncCtrlW.queue, pMsg); + case SYNC_RD_QUEUE: + dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pSyncRdW.queue, pMsg); break; case APPLY_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg); @@ -234,9 +234,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp return code; } -int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE); -} +int32_t vmPutMsgToSyncRdQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_RD_QUEUE); } int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); } @@ -327,18 +325,18 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl}; SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; - SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-ctrl", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; + SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode->pImpl}; (void)tMultiWorkerInit(&pVnode->pWriteW, &wcfg); (void)tMultiWorkerInit(&pVnode->pSyncW, &scfg); - (void)tMultiWorkerInit(&pVnode->pSyncCtrlW, &sccfg); + (void)tMultiWorkerInit(&pVnode->pSyncRdW, &sccfg); (void)tMultiWorkerInit(&pVnode->pApplyW, &acfg); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); pVnode->pStreamQ = tAutoQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue); pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue); - if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncCtrlW.queue == NULL || + if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL || pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -348,8 +346,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pWriteW.queue->threadId); dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue, pVnode->pSyncW.queue->threadId); - dInfo("vgId:%d, sync-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlW.queue, - pVnode->pSyncCtrlW.queue->threadId); + dInfo("vgId:%d, sync-rd-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue, + pVnode->pSyncRdW.queue->threadId); dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, pVnode->pApplyW.queue->threadId); dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 18548db56f..4965d5c34a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -33,7 +33,7 @@ static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { return -1; } - int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg); + int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b49ca70bfa..d681f5b65e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -378,7 +378,7 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { return -1; } - int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg); + int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; From 9f97162ef77956cfa3feb304d815e2867b60d589 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 14 Mar 2023 21:02:06 +0800 Subject: [PATCH 007/176] enh: separate election timer events --- include/common/tmsgdef.h | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 3 ++- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 3 ++- source/libs/sync/src/syncMain.c | 3 +++ source/libs/sync/src/syncMessage.c | 4 ++-- source/libs/sync/src/syncTimeout.c | 5 ----- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 46ca814e50..96d18d1abc 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -259,7 +259,7 @@ enum { TD_NEW_MSG_SEG(TDMT_SYNC_MSG) TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_SYNC_PING, "sync-ping", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT_ELECTION, "sync-elect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) // no longer used TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST, "sync-client-request", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST_BATCH, "sync-client-request-batch", NULL, NULL) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index c129ea21ac..ce485cb6ca 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -188,17 +188,18 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DISABLE_WRITE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index b60d2c3caf..84dfab7e6c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -549,10 +549,12 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; @@ -562,7 +564,6 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ed97c8abde..a617a5d293 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -182,6 +182,9 @@ int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) { case TDMT_SYNC_TIMEOUT: code = syncNodeOnTimeout(pSyncNode, pMsg); break; + case TDMT_SYNC_TIMEOUT_ELECTION: + code = syncNodeOnTimeout(pSyncNode, pMsg); + break; case TDMT_SYNC_CLIENT_REQUEST: code = syncNodeOnClientRequest(pSyncNode, pMsg, NULL); break; diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 2a44588eef..72c8887803 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -22,7 +22,7 @@ int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType timeoutType, uint64_t l SSyncNode* pNode) { int32_t bytes = sizeof(SyncTimeout); pMsg->pCont = rpcMallocCont(bytes); - pMsg->msgType = TDMT_SYNC_TIMEOUT; + pMsg->msgType = (timeoutType == SYNC_TIMEOUT_ELECTION) ? TDMT_SYNC_TIMEOUT_ELECTION : TDMT_SYNC_TIMEOUT; pMsg->contLen = bytes; if (pMsg->pCont == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -31,7 +31,7 @@ int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType timeoutType, uint64_t l SyncTimeout* pTimeout = pMsg->pCont; pTimeout->bytes = bytes; - pTimeout->msgType = TDMT_SYNC_TIMEOUT; + pTimeout->msgType = pMsg->msgType; pTimeout->vgId = pNode->vgId; pTimeout->timeoutType = timeoutType; pTimeout->logicClock = logicClock; diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 859183db95..a27be2853e 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -120,9 +120,6 @@ int32_t syncNodeOnTimeout(SSyncNode* ths, const SRpcMsg* pRpc) { if (atomic_load_64(&ths->pingTimerLogicClockUser) <= pMsg->logicClock) { ++(ths->pingTimerCounter); - // syncNodePingAll(ths); - // syncNodePingPeers(ths); - syncNodeTimerRoutine(ths); } @@ -138,8 +135,6 @@ int32_t syncNodeOnTimeout(SSyncNode* ths, const SRpcMsg* pRpc) { ++(ths->heartbeatTimerCounter); sTrace("vgId:%d, sync timer, type:replicate count:%" PRIu64 ", lc-user:%" PRIu64, ths->vgId, ths->heartbeatTimerCounter, ths->heartbeatTimerLogicClockUser); - - // syncNodeReplicate(ths, true); } } else { From 497ba9992d5c2e1b542d5951ab9088edd8256cf7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Mar 2023 02:26:18 +0000 Subject: [PATCH 008/176] fix coverity scan problem --- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndIndex.c | 4 ++-- source/libs/index/src/indexFstFile.c | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index dfc3b3fde8..fead246ed6 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -392,7 +392,7 @@ typedef struct { } SSmaObj; typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; + char name[TSDB_INDEX_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; char dstTbName[TSDB_TABLE_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 8782fd823f..83172acf64 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -138,7 +138,7 @@ static void *mndBuildDropIdxReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStbOb mInfo("idx: %s start to build drop index req", pIdx->name); len = tSerializeSDropIdxReq(NULL, 0, &req); - if (ret < 0) { + if (len < 0) { goto _err; } @@ -672,7 +672,7 @@ _OVER: static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; SIdxObj idxObj = {0}; - memcpy(idxObj.name, req->idxName, TSDB_TABLE_FNAME_LEN); + memcpy(idxObj.name, req->idxName, TSDB_INDEX_FNAME_LEN); memcpy(idxObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN); memcpy(idxObj.db, pDb->name, TSDB_DB_FNAME_LEN); memcpy(idxObj.colName, req->colName, TSDB_COL_NAME_LEN); diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 40c50ed9cb..9e7ed52104 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -127,8 +127,6 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of blk->blockId = blkId; blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); ASSERTS(blk->nread <= kBlockSize, "index read incomplete data"); - if (blk->nread > kBlockSize) break; - if (blk->nread < kBlockSize && blk->nread < len) { taosMemoryFree(blk); break; From 7616a283e3f1376bc9350399fbac55c649d893b2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 18 Mar 2023 18:23:38 +0800 Subject: [PATCH 009/176] fix:error in TD-23218 & remove useless logic --- include/common/tcommon.h | 1 - include/libs/executor/executor.h | 4 +- include/libs/wal/wal.h | 4 +- include/util/taoserror.h | 1 + source/dnode/vnode/inc/vnode.h | 12 +- source/dnode/vnode/src/tq/tq.c | 183 +++++++++- source/dnode/vnode/src/tq/tqExec.c | 63 +--- source/dnode/vnode/src/tq/tqRead.c | 432 ++---------------------- source/libs/executor/inc/executorimpl.h | 6 +- source/libs/executor/src/executor.c | 18 +- source/libs/executor/src/executorimpl.c | 9 + source/libs/executor/src/scanoperator.c | 98 ++---- source/libs/wal/src/walRead.c | 17 +- source/util/src/terror.c | 1 + 14 files changed, 283 insertions(+), 566 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 2a40976a8b..2614864b16 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -216,7 +216,6 @@ typedef struct SSDataBlock { enum { FETCH_TYPE__DATA = 1, - FETCH_TYPE__META, FETCH_TYPE__SEP, FETCH_TYPE__NONE, }; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index c3d2010351..dcc3c86171 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -196,12 +196,10 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); -int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); +void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); -int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo); - const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo); const char* qExtractTbnameFromTask(qTaskInfo_t tinfo); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 014ed518a3..ccbc53fa5d 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -146,8 +146,8 @@ typedef struct { int64_t curFileFirstVer; int64_t curVersion; int64_t capacity; - int8_t curInvalid; - int8_t curStopped; +// int8_t curInvalid; +// int8_t curStopped; TdThreadMutex mutex; SWalFilterCond cond; // TODO remove it diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 3dc3aa69b7..87eb3b3deb 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -756,6 +756,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) #define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002) +#define TSDB_CODE_TMQ_CONSUMER_ERROR TAOS_DEF_ERROR_CODE(0, 0x4003) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 1d14829891..f3d983ae46 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -230,12 +230,7 @@ typedef struct SSnapContext { } SSnapContext; typedef struct STqReader { - // const SSubmitReq *pMsg; - // SSubmitBlk *pBlock; - // SSubmitMsgIter msgIter; - // SSubmitBlkIter blkIter; - - int64_t ver; +// int64_t ver; SPackedData msg2; int8_t setMsg; @@ -264,8 +259,13 @@ int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); +<<<<<<< Updated upstream int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char* id); int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); +======= +int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +void tqNextBlock(STqReader *pReader, SFetchRet *ret); +>>>>>>> Stashed changes int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); // int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index cd67dc23ad..334508eecf 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -211,6 +211,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { return 0; } +<<<<<<< Updated upstream int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) { int32_t len = 0; int32_t code = 0; @@ -242,6 +243,12 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con .code = 0, }; tmsgSendRsp(&rsp); +======= +int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { + SMqDataRsp* pRsp = pPushEntry->pDataRsp; + SMqRspHead* pHeader = &pPushEntry->pDataRsp->head; + doSendDataRsp(&pPushEntry->info, pRsp, pHeader->epoch, pHeader->consumerId, pHeader->mqMsgType); +>>>>>>> Stashed changes char buf1[80] = {0}; char buf2[80] = {0}; @@ -253,6 +260,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con return 0; } +<<<<<<< Updated upstream int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) { int32_t len = 0; int32_t code = 0; @@ -284,6 +292,10 @@ int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, co .code = 0, }; tmsgSendRsp(&rsp); +======= +int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type) { + doSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type); +>>>>>>> Stashed changes char buf1[80] = {0}; char buf2[80] = {0}; @@ -435,6 +447,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } +<<<<<<< Updated upstream // update epoch if need int32_t savedEpoch = atomic_load_32(&pHandle->epoch); while (savedEpoch < reqEpoch) { @@ -552,16 +565,62 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } // for taosx +======= +static int32_t processSubColumn(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) { + int32_t vgId = TD_VID(pTq->pVnode); + + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); + + // lock + taosWLockLatch(&pTq->pushLock); + int code = tqScanData(pTq, pHandle, &dataRsp, offset); + if(code != 0) { + tDeleteSMqDataRsp(&dataRsp); + return TSDB_CODE_TMQ_CONSUMER_ERROR; + } + // till now, all data has been transferred to consumer, new data needs to push client once arrived. + if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && + dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { + code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + taosWUnLockLatch(&pTq->pushLock); + return code; + } + + taosWUnLockLatch(&pTq->pushLock); + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); + + // NOTE: this pHandle->consumerId may have been changed already. + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64 + ", ts:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid, + dataRsp.rspOffset.ts); + + tDeleteSMqDataRsp(&dataRsp); + return code; +} + +static int32_t processSubDbOrTable(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) { + int code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + SWalCkHead* pCkHead = NULL; +>>>>>>> Stashed changes SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; tqInitTaosxRsp(&taosxRsp, &req); +<<<<<<< Updated upstream if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew) < 0) { +======= + if (offset->type != TMQ_OFFSET__LOG) { + if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { +>>>>>>> Stashed changes return -1; } if (metaRsp.metaRspLen > 0) { +<<<<<<< Updated upstream if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { code = -1; } @@ -569,17 +628,29 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { ",version:%" PRId64, consumerId, pHandle->subKey, TD_VID(pTq->pVnode), metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.version); +======= + code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp); + tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 + ",ts:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, + metaRsp.rspOffset.ts); +>>>>>>> Stashed changes taosMemoryFree(metaRsp.metaRsp); tDeleteSTaosxRsp(&taosxRsp); return code; } + tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 + ",version:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid, + taosxRsp.rspOffset.version); if (taosxRsp.blockNum > 0) { if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } tDeleteSTaosxRsp(&taosxRsp); return code; +<<<<<<< Updated upstream } else { fetchOffsetNew = taosxRsp.rspOffset; } @@ -592,6 +663,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (fetchOffsetNew.type == TMQ_OFFSET__LOG) { int64_t fetchVer = fetchOffsetNew.version + 1; +======= + } + } else { + int64_t fetchVer = offset->version + 1; +>>>>>>> Stashed changes pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { tDeleteSTaosxRsp(&taosxRsp); @@ -601,28 +677,45 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { walSetReaderCapacity(pHandle->pWalReader, 2048); while (1) { +<<<<<<< Updated upstream savedEpoch = atomic_load_32(&pHandle->epoch); if (savedEpoch > reqEpoch) { tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, vg %d offset %" PRId64 ", found new consumer epoch %d, discard req epoch %d", consumerId, req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, savedEpoch, reqEpoch); +======= + int32_t savedEpoch = atomic_load_32(&pHandle->epoch); + if (savedEpoch > pRequest->epoch) { + tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64 + ", found new consumer epoch %d, discard req epoch %d", + pRequest->consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch); +>>>>>>> Stashed changes break; } if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); +<<<<<<< Updated upstream if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } +======= + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); +>>>>>>> Stashed changes tDeleteSTaosxRsp(&taosxRsp); taosMemoryFreeClear(pCkHead); return code; } SWalCont* pHead = &pCkHead->head; +<<<<<<< Updated upstream tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId, req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); +======= + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", pRequest->consumerId, + pRequest->epoch, vgId, fetchVer, pHead->msgType); +>>>>>>> Stashed changes if (pHead->msgType == TDMT_VND_SUBMIT) { SPackedData submit = { @@ -631,8 +724,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { .ver = pHead->version, }; if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) { +<<<<<<< Updated upstream tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, TD_VID(pTq->pVnode), req.subKey); +======= + tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, + pRequest->subKey); +>>>>>>> Stashed changes return -1; } if (taosxRsp.blockNum > 0 /* threshold */) { @@ -648,8 +746,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } } else { - /*A(pHandle->fetchMeta);*/ - /*A(IS_META_MSG(pHead->msgType));*/ tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); metaRsp.resMsgType = pHead->msgType; @@ -674,6 +770,89 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return 0; } +<<<<<<< Updated upstream +======= +static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { + int32_t code = -1; + STqOffsetVal offset = {0}; + STqOffsetVal reqOffset = pRequest->reqOffset; + + // 1. reset the offset if needed + if (reqOffset.type > 0) { + offset = reqOffset; + } else { // handle the reset offset cases, according to the consumer's choice. + bool blockReturned = false; + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); + if (code != 0) { + return code; + } + + // empty block returned, quit + if (blockReturned) { + return 0; + } + } + + // this is a normal subscription requirement + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + return processSubColumn(pTq, pHandle, pRequest, pMsg, &offset); + } + + // todo handle the case where re-balance occurs. + // for taosx + return processSubDbOrTable(pTq, pHandle, pRequest, pMsg, &offset); +} + +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { + SMqPollReq req = {0}; + if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int64_t consumerId = req.consumerId; + int32_t reqEpoch = req.epoch; + STqOffsetVal reqOffset = req.reqOffset; + int32_t vgId = TD_VID(pTq->pVnode); + + // 1. find handle + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle == NULL) { + tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + // 2. check re-balance status + taosRLockLatch(&pTq->pushLock); + if (pHandle->consumerId != consumerId) { + tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosRUnLockLatch(&pTq->pushLock); + return -1; + } + taosRUnLockLatch(&pTq->pushLock); + + taosWLockLatch(&pTq->pushLock); + // 3. update the epoch value + int32_t savedEpoch = pHandle->epoch; + if (savedEpoch < reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, reqEpoch); + pHandle->epoch = reqEpoch; + } + taosWUnLockLatch(&pTq->pushLock); + + char buf[80]; + tFormatOffset(buf, 80, &reqOffset); + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, + consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); + + return extractDataForMq(pTq, pHandle, &req, pMsg); +} + +>>>>>>> Stashed changes int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index f7eba3fbc1..36f4d448c8 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -65,18 +65,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs qTaskInfo_t task = pExec->task; if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - if (pOffset->type == TMQ_OFFSET__LOG) { - pRsp->rspOffset = *pOffset; - return 0; - } else { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer); - if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - pRsp->rspOffset = *pOffset; - return 0; - } - } + tqError("prepare scan failed, return"); + return -1; } int32_t rowCnt = 0; @@ -103,20 +93,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs } } - if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { - return -1; - } - - if (pRsp->rspOffset.type == 0) { - tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, - pRsp->rspOffset.uid, pRsp->rspOffset.version); - return -1; - } - - if(pRsp->withTbName || pRsp->withSchema){ - tqError("get column should not with meta:%d,%d", pRsp->withTbName, pRsp->withSchema); - return -1; - } + qStreamExtractOffset(task, &pRsp->rspOffset); return 0; } @@ -125,18 +102,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta qTaskInfo_t task = pExec->task; if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - if (pOffset->type == TMQ_OFFSET__LOG) { - pRsp->rspOffset = *pOffset; - return 0; - } else { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer); - if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - pRsp->rspOffset = *pOffset; - return 0; - } - } + tqDebug("tqScanTaosx prepare scan failed, return"); + return -1; } int32_t rowCnt = 0; @@ -183,9 +150,6 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - if (qStreamExtractPrepareUid(task) != 0) { - continue; - } tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); break; @@ -198,28 +162,17 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta SMqMetaRsp* tmp = qStreamExtractMetaMsg(task); if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts); - qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); - tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META; tqDebug("tmqsnap task exec change to get data"); continue; } *pMetaRsp = *tmp; tqDebug("tmqsnap task exec exited, get meta"); - - tqDebug("task exec exited"); break; } qStreamExtractOffset(task, &pRsp->rspOffset); - if (pRsp->rspOffset.type == 0) { - tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, - pRsp->rspOffset.uid, pRsp->rspOffset.version); - return -1; - } - return 0; } @@ -232,14 +185,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { STqReader* pReader = pExec->pExecReader; - /*tqReaderSetDataMsg(pReader, pReq, 0);*/ tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); while (tqNextDataBlock2(pReader)) { - /*SSDataBlock block = {0};*/ - /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ - /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ - /*}*/ - taosArrayClear(pBlocks); taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index bf73cca925..14623730e1 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -260,7 +260,7 @@ STqReader* tqOpenReader(SVnode* pVnode) { pReader->pVnodeMeta = pVnode->pMeta; /*pReader->pMsg = NULL;*/ - pReader->ver = -1; +// pReader->ver = -1; pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; pReader->cachedSchemaSuid = 0; @@ -291,35 +291,24 @@ void tqCloseReader(STqReader* pReader) { } int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { - // todo set the correct vgId - tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id); if (walReadSeekVer(pReader->pWalReader, ver) < 0) { tqError("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id); return -1; - } else { - tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id); - return 0; } + tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id); + return 0; } -int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { - bool fromProcessedMsg = pReader->msg2.msgStr != NULL; - +void tqNextBlock(STqReader* pReader, SFetchRet* ret) { while (1) { - if (!fromProcessedMsg) { + if (pReader->msg2.msgStr == NULL) { if (walNextValidMsg(pReader->pWalReader) < 0) { -// pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped; - if(pReader->pWalReader->curInvalid == 0){ - pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped; - }else{ - pReader->ver = walGetLastVer(pReader->pWalReader->pWal); - } +// pReader->ver = pReader->pWalReader->curVersion; ret->offset.type = TMQ_OFFSET__LOG; - - ret->offset.version = pReader->ver; + ret->offset.version = pReader->pWalReader->curVersion; ret->fetchType = FETCH_TYPE__NONE; - tqDebug("return offset %" PRId64 ", no more valid msg in wal", ret->offset.version); - return -1; + tqInfo("return offset %" PRId64 ", no more valid msg in wal", ret->offset.version); + return; } void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); @@ -330,7 +319,6 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { } while (tqNextDataBlock2(pReader)) { - // TODO mem free memset(&ret->data, 0, sizeof(SSDataBlock)); int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL); if (code != 0 || ret->data.info.rows == 0) { @@ -338,45 +326,18 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { } ret->fetchType = FETCH_TYPE__DATA; tqDebug("return data rows %d", ret->data.info.rows); - return 0; - } - - if (fromProcessedMsg) { - ret->offset.type = TMQ_OFFSET__LOG; - ret->offset.version = pReader->ver; - ret->fetchType = FETCH_TYPE__SEP; - tqDebug("return offset %" PRId64 ", processed finish", ret->offset.version); - return 0; + return; } } } -#if 0 -int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) { - pReader->pMsg = pMsg; - -// if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; -// while (true) { -// if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) return -1; -// tqDebug("submitnext vgId:%d, block:%p, dataLen:%d, len:%d, uid:%"PRId64, pWalReader->pWal->cfg.vgId, pReader->pBlock, pReader->msgIter.dataLen, -// pReader->msgIter.len, pReader->msgIter.uid); -// if (pReader->pBlock == NULL) break; -// } - - if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; - pReader->ver = ver; - memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); - return 0; -} -#endif - int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { - ASSERT(pReader->msg2.msgStr == NULL && msgStr && msgLen && (ver >= 0)); +// ASSERT(pReader->msg2.msgStr == NULL && msgStr && msgLen && (ver >= 0)); pReader->msg2.msgStr = msgStr; pReader->msg2.msgLen = msgLen; pReader->msg2.ver = ver; - pReader->ver = ver; +// pReader->ver = ver; tqDebug("tq reader set msg %p %d", msgStr, msgLen); @@ -384,7 +345,9 @@ int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, SDecoder decoder; tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { - ASSERT(0); + tDecoderClear(&decoder); + tqError("DecodeSSubmitReq2 error, msgLen:%d, ver:%"PRId64, msgLen, ver); + return -1; } tDecoderClear(&decoder); pReader->setMsg = 1; @@ -392,47 +355,17 @@ int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, return 0; } -#if 0 -bool tqNextDataBlock(STqReader* pReader) { - if (pReader->pMsg == NULL) return false; - while (1) { - if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) { - return false; - } - if (pReader->pBlock == NULL) { - pReader->pMsg = NULL; - return false; - } - - if (pReader->tbIdHash == NULL) { - return true; - } - void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t)); - /*tqDebug("search uid %" PRId64, pHandle->msgIter.uid);*/ - if (ret != NULL) { - /*tqDebug("find uid %" PRId64, pHandle->msgIter.uid);*/ - return true; - } - } - return false; -} -#endif - bool tqNextDataBlock2(STqReader* pReader) { - if (pReader->msg2.msgStr == NULL) { + if (pReader->msg2.msgStr == NULL || pReader->setMsg != 1) { return false; } - ASSERT(pReader->setMsg == 1); - tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, pReader->msg2.ver, pReader->nextBlk); int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - ASSERT(pSubmitTbData->uid); - if (pReader->tbIdHash == NULL) return true; void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); @@ -503,63 +436,8 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } -#if 0 -bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { - while (1) { - if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { - return false; - } - if (pHandle->pBlock == NULL) return false; - - void* ret = taosHashGet(filterOutUids, &pHandle->msgIter.uid, sizeof(int64_t)); - if (ret == NULL) { - return true; - } - } - return false; -} - -int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) { - // - int32_t sversion = htonl(pReader->pBlock->sversion); - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || - pReader->cachedSchemaSuid != pReader->msgIter.suid) { - taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 - "), version %d, possibly dropped table", - pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - tDeleteSSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchemaWrapper == NULL) { - tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - STSchema* pTschema = pReader->pSchema; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - - int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); - } - return 0; -} -#endif - int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) { - int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); - ASSERT(pReader->nextBlk < blockSz); - - tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); - + tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg2.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); pReader->nextBlk++; @@ -663,33 +541,27 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD int32_t targetIdx = 0; int32_t sourceIdx = 0; while (targetIdx < colActual) { - ASSERT(sourceIdx < numOfCols); - + if(sourceIdx >= numOfCols){ + tqError("tqRetrieveDataBlock2 sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols); + goto FAIL; + } SColData* pCol = taosArrayGet(pCols, sourceIdx); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); SColVal colVal; - ASSERT(pCol->nVal == numOfRows); + if(pCol->nVal != numOfRows){ + tqError("tqRetrieveDataBlock2 pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); + goto FAIL; + } if (pCol->cid < pColData->info.colId) { sourceIdx++; } else if (pCol->cid == pColData->info.colId) { for (int32_t i = 0; i < pCol->nVal; i++) { tColDataGetValue(pCol, i, &colVal); -#if 0 - void* val = NULL; - if (IS_STR_DATA_TYPE(colVal.type)) { - val = colVal.value.pData; - } else { - val = &colVal.value.val; - } - if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - goto FAIL; - } -#endif if (IS_STR_DATA_TYPE(colVal.type)) { if (colVal.value.pData != NULL) { - char val[65535 + 2]; + char val[65535 + 2] = {0}; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { @@ -720,8 +592,6 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD for (int32_t j = 0; j < colActual; j++) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); while (1) { - ASSERT(sourceIdx < pTschema->numOfCols); - SColVal colVal; tRowGet(pRow, pTschema, sourceIdx, &colVal); if (colVal.cid < pColData->info.colId) { @@ -730,7 +600,7 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD } else if (colVal.cid == pColData->info.colId) { if (IS_STR_DATA_TYPE(colVal.type)) { if (colVal.value.pData != NULL) { - char val[65535 + 2]; + char val[65535 + 2] = {0}; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { @@ -739,7 +609,6 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD } else { colDataSetNULL(pColData, i); } - /*val = colVal.value.pData;*/ } else { if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; @@ -764,253 +633,6 @@ FAIL: return -1; } -#if 0 -int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { - // TODO: cache multiple schema - int32_t sversion = htonl(pReader->pBlock->sversion); - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || - pReader->cachedSchemaSuid != pReader->msgIter.suid) { - if (pReader->pSchema) taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", - pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchemaWrapper == NULL) { - tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pReader->msgIter.uid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - pReader->cachedSchemaVer = sversion; - pReader->cachedSchemaSuid = pReader->msgIter.suid; - } - - STSchema* pTschema = pReader->pSchema; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - - int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); - - if (colNumNeed == 0) { - int32_t colMeta = 0; - while (colMeta < pSchemaWrapper->nCols) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - goto FAIL; - } - colMeta++; - } - } else { - if (colNumNeed > pSchemaWrapper->nCols) { - colNumNeed = pSchemaWrapper->nCols; - } - - int32_t colMeta = 0; - int32_t colNeed = 0; - while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - col_id_t colIdSchema = pColSchema->colId; - col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); - if (colIdSchema < colIdNeed) { - colMeta++; - } else if (colIdSchema > colIdNeed) { - colNeed++; - } else { - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - goto FAIL; - } - colMeta++; - colNeed++; - } - } - } - - if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - int32_t colActual = blockDataGetNumOfCols(pBlock); - - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pTschema); - STSRow* row; - int32_t curRow = 0; - - tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter); - - pBlock->info.id.uid = pReader->msgIter.uid; - pBlock->info.rows = pReader->msgIter.numOfRows; - pBlock->info.version = pReader->pMsg->version; - pBlock->info.dataLoad = 1; - - while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { - tdSTSRowIterReset(&iter, row); - // get all wanted col of that block - for (int32_t i = 0; i < colActual; i++) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - SCellVal sVal = {0}; - if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { - break; - } - if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { - goto FAIL; - } - } - curRow++; - } - return 0; - -FAIL: - blockDataFreeRes(pBlock); - return -1; -} - -int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas) { - int32_t sversion = htonl(pReader->pBlock->sversion); - - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || - pReader->cachedSchemaSuid != pReader->msgIter.suid) { - if (pReader->pSchema) taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", - pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchemaWrapper == NULL) { - tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pReader->msgIter.uid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - pReader->cachedSchemaVer = sversion; - pReader->cachedSchemaSuid = pReader->msgIter.suid; - } - - STSchema* pTschema = pReader->pSchema; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - - int32_t colAtMost = pSchemaWrapper->nCols; - - int32_t curRow = 0; - int32_t lastRow = 0; - - char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); - if (assigned == NULL) return -1; - - tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter); - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pTschema); - STSRow* row; - - while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { - bool buildNew = false; - tdSTSRowIterReset(&iter, row); - - tqDebug("vgId:%d, row of block %d", pReader->pWalReader->pWal->cfg.vgId, curRow); - for (int32_t i = 0; i < colAtMost; i++) { - SCellVal sVal = {0}; - if (!tdSTSRowIterFetch(&iter, pSchemaWrapper->pSchema[i].colId, pSchemaWrapper->pSchema[i].type, &sVal)) { - break; - } - tqDebug("vgId:%d, %d col, type %d", pReader->pWalReader->pWal->cfg.vgId, i, sVal.valType); - if (curRow == 0) { - assigned[i] = sVal.valType != TD_VTYPE_NONE; - buildNew = true; - } else { - bool currentRowAssigned = sVal.valType != TD_VTYPE_NONE; - if (currentRowAssigned != assigned[i]) { - assigned[i] = currentRowAssigned; - buildNew = true; - } - } - } - - if (buildNew) { - if (taosArrayGetSize(blocks) > 0) { - SSDataBlock* pLastBlock = taosArrayGetLast(blocks); - pLastBlock->info.rows = curRow - lastRow; - lastRow = curRow; - } - SSDataBlock* pBlock = createDataBlock(); - SSchemaWrapper* pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - if (tqMaskBlock(pSW, pBlock, pSchemaWrapper, assigned) < 0) { - blockDataDestroy(pBlock); - goto FAIL; - } - SSDataBlock block = {0}; - assignOneDataBlock(&block, pBlock); - blockDataDestroy(pBlock); - - tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId, - (int32_t)taosArrayGetSize(block.pDataBlock)); - - taosArrayPush(blocks, &block); - taosArrayPush(schemas, &pSW); - } - - SSDataBlock* pBlock = taosArrayGetLast(blocks); - pBlock->info.id.uid = pReader->msgIter.uid; - pBlock->info.rows = 0; - pBlock->info.version = pReader->pMsg->version; - - tqDebug("vgId:%d, taosx scan, block num: %d", pReader->pWalReader->pWal->cfg.vgId, - (int32_t)taosArrayGetSize(blocks)); - - if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows - curRow) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - tdSTSRowIterReset(&iter, row); - for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); i++) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - SCellVal sVal = {0}; - - if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { - break; - } - - ASSERT(sVal.valType != TD_VTYPE_NONE); - - if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { - goto FAIL; - } - tqDebug("vgId:%d, row %d col %d append %d", pReader->pWalReader->pWal->cfg.vgId, curRow, i, - sVal.valType == TD_VTYPE_NULL); - } - curRow++; - } - SSDataBlock* pLastBlock = taosArrayGetLast(blocks); - pLastBlock->info.rows = curRow - lastRow; - - taosMemoryFree(assigned); - return 0; - -FAIL: - taosMemoryFree(assigned); - return -1; -} -#endif - int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 55bf8b37d0..27355b5893 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -127,10 +127,10 @@ enum { typedef struct { // TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq +// STqOffsetVal prepareStatus; // for tmq + STqOffsetVal currentOffset; // for tmq SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; +// int8_t returned; int64_t snapshotVer; // const SSubmitReq* pReq; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index af625be0b1..af7fad8a59 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -995,15 +995,9 @@ SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { return &pTaskInfo->streamInfo.metaRsp; } -int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo) { +void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return pTaskInfo->streamInfo.prepareStatus.uid; -} - -int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); - return 0; + memcpy(pOffset, &pTaskInfo->streamInfo.currentOffset, sizeof(STqOffsetVal)); } int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) { @@ -1052,21 +1046,19 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; - pTaskInfo->streamInfo.prepareStatus = *pOffset; - pTaskInfo->streamInfo.returned = 0; - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) { + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } + pTaskInfo->streamInfo.currentOffset = *pOffset; if (subType == TOPIC_SUB_TYPE__COLUMN) { pOperator->status = OP_OPENED; - // TODO add more check if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if(pOperator->numOfDownstream != 1){ qError("pOperator->numOfDownstream != 1:%d", pOperator->numOfDownstream); - return -1; + return TSDB_CODE_TMQ_CONSUMER_ERROR; } pOperator = pOperator->pDownstream[0]; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0dcefec93d..fdfb94a073 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2774,9 +2774,18 @@ void qStreamCloseTsdbReader(void* task) { if (task == NULL) return; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task; SOperatorInfo* pOp = pTaskInfo->pRoot; +<<<<<<< Updated upstream qDebug("stream close tsdb reader, reset status uid %" PRId64 " ts %" PRId64, pTaskInfo->streamInfo.lastStatus.uid, pTaskInfo->streamInfo.lastStatus.ts); pTaskInfo->streamInfo.lastStatus = (STqOffsetVal){0}; +======= + + qDebug("stream close tsdb reader, reset status uid:%" PRId64 " ts:%" PRId64, pTaskInfo->streamInfo.currentOffset.uid, + pTaskInfo->streamInfo.currentOffset.ts); + + // todo refactor, other thread may already use this read to extract data. + pTaskInfo->streamInfo.currentOffset = (STqOffsetVal){0}; +>>>>>>> Stashed changes while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) { SOperatorInfo* pDownstreamOp = pOp->pDownstream[0]; if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 40b9597643..98386b5496 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -673,9 +673,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { // todo refactor /*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/ /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/ - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; - pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; - pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; +// pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; +// pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; +// pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; ASSERT(pBlock->info.id.uid != 0); return pBlock; @@ -853,9 +853,11 @@ static void destroyTableScanOperatorInfo(void* param) { SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { + int32_t code = 0; STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } @@ -863,8 +865,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; int32_t numOfCols = 0; - int32_t code = - extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo); + code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1567,17 +1568,10 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.submit.msgStr != NULL) { if (pInfo->tqReader->msg2.msgStr == NULL) { - /*pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq;*/ - - /*const SSubmitReq* pSubmit = pInfo->tqReader->pMsg;*/ - /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ - /*void* msgStr = pTaskInfo->streamInfo.*/ SPackedData submit = pTaskInfo->streamInfo.submit; if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { qError("submit msg messed up when initing stream submit block %p", submit.msgStr); - pInfo->tqReader->msg2 = (SPackedData){0}; - pInfo->tqReader->setMsg = 0; - ASSERT(0); + return NULL; } } @@ -1605,13 +1599,14 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { return NULL; } - if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion); - pTaskInfo->streamInfo.returned = 1; + tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); return pResult; +<<<<<<< Updated upstream } else { if (!pTaskInfo->streamInfo.returned) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; @@ -1626,45 +1621,35 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } else { return NULL; } +======= + } + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pTaskInfo->streamInfo.snapshotVer); + qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); + if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { + return NULL; +>>>>>>> Stashed changes } } - if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__LOG) { + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { while (1) { SFetchRet ret = {0}; - if (tqNextBlock(pInfo->tqReader, &ret) < 0) { - // if the end is reached, terrno is 0 - if (terrno != 0) { - qError("failed to get next log block since %s", terrstr()); - } - } + tqNextBlock(pInfo->tqReader, &ret); + pTaskInfo->streamInfo.currentOffset = ret.offset; if (ret.fetchType == FETCH_TYPE__DATA) { blockDataCleanup(pInfo->pRes); setBlockIntoRes(pInfo, &ret.data, true); - if (pInfo->pRes->info.rows > 0) { - pOperator->status = OP_EXEC_RECV; - qDebug("queue scan log return %d rows", pInfo->pRes->info.rows); - return pInfo->pRes; - } - } else if (ret.fetchType == FETCH_TYPE__META) { - qError("unexpected ret.fetchType:%d", ret.fetchType); - continue; - // pTaskInfo->streamInfo.lastStatus = ret.offset; - // pTaskInfo->streamInfo.metaBlk = ret.meta; - // return NULL; - } else if (ret.fetchType == FETCH_TYPE__NONE || - (ret.fetchType == FETCH_TYPE__SEP && pOperator->status == OP_EXEC_RECV)) { - pTaskInfo->streamInfo.lastStatus = ret.offset; - char formatBuf[80]; - tFormatOffset(formatBuf, 80, &ret.offset); - qDebug("queue scan log return null, offset %s", formatBuf); - pOperator->status = OP_OPENED; - return NULL; + qDebug("queue scan log return %d rows", pInfo->pRes->info.rows); + return pInfo->pRes; } + return NULL; } } else { - qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.prepareStatus.type); + qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.currentOffset.type); return NULL; } } @@ -2094,7 +2079,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.metaRsp.metaRsp = NULL; qDebug("tmqsnap doRawScan called"); - if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) { if (isTaskKilled(pTaskInfo)) { tsdbReleaseDataBlock(pInfo->dataReader); @@ -2107,28 +2092,22 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { } qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.id.uid); - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; - pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; - pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; + tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pBlock->info.id.uid, pBlock->info.window.ekey); return pBlock; } SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); - pTaskInfo->streamInfo.prepareStatus.uid = mtInfo.uid; - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG; - pTaskInfo->streamInfo.lastStatus.version = pInfo->sContext->snapVersion; + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->sContext->snapVersion); } else { - pTaskInfo->streamInfo.prepareStatus.uid = mtInfo.uid; - pTaskInfo->streamInfo.prepareStatus.ts = INT64_MIN; + tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, mtInfo.uid, INT64_MIN); qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); - qStreamPrepareScan(pTaskInfo, &pTaskInfo->streamInfo.prepareStatus, pInfo->sContext->subType); } tDeleteSSchemaWrapper(mtInfo.schema); qDebug("tmqsnap stream scan tsdb return null"); return NULL; - } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_META) { + } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) { SSnapContext* sContext = pInfo->sContext; void* data = NULL; int32_t dataLen = 0; @@ -2141,15 +2120,11 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { } if (!sContext->queryMetaOrData) { // change to get data next poll request - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_META; - pTaskInfo->streamInfo.lastStatus.uid = uid; - pTaskInfo->streamInfo.metaRsp.rspOffset.type = TMQ_OFFSET__SNAPSHOT_DATA; - pTaskInfo->streamInfo.metaRsp.rspOffset.uid = 0; - pTaskInfo->streamInfo.metaRsp.rspOffset.ts = INT64_MIN; + tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, 0, INT64_MIN); + pTaskInfo->streamInfo.metaRsp.rspOffset = pTaskInfo->streamInfo.currentOffset; } else { - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_META; - pTaskInfo->streamInfo.lastStatus.uid = uid; - pTaskInfo->streamInfo.metaRsp.rspOffset = pTaskInfo->streamInfo.lastStatus; + tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid); + pTaskInfo->streamInfo.metaRsp.rspOffset = pTaskInfo->streamInfo.currentOffset; pTaskInfo->streamInfo.metaRsp.resMsgType = type; pTaskInfo->streamInfo.metaRsp.metaRspLen = dataLen; pTaskInfo->streamInfo.metaRsp.metaRsp = data; @@ -2351,7 +2326,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->base.dataReader = NULL; - pTaskInfo->streamInfo.lastStatus.uid = -1; } if (pHandle->initTqReader) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 3e1e36ccc1..ad6127ead2 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -33,7 +33,6 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { pReader->pLogFile = NULL; pReader->curVersion = -1; pReader->curFileFirstVer = -1; - pReader->curInvalid = 1; pReader->capacity = 0; if (cond) { pReader->cond = *cond; @@ -81,7 +80,6 @@ int32_t walNextValidMsg(SWalReader *pReader) { wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 ", applied index:%" PRId64 ", end index:%" PRId64, pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer); - pReader->curStopped = 0; while (fetchVer <= endVer) { if (walFetchHeadNew(pReader, fetchVer) < 0) { return -1; @@ -99,7 +97,6 @@ int32_t walNextValidMsg(SWalReader *pReader) { fetchVer = pReader->curVersion; } } - pReader->curStopped = 1; return -1; } @@ -196,17 +193,16 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { return -1; } - wDebug("vgId:%d, wal version reset from %" PRId64 "(invalid:%d) to %" PRId64, pReader->pWal->cfg.vgId, - pReader->curVersion, pReader->curInvalid, ver); + wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, + pReader->curVersion, ver); pReader->curVersion = ver; - pReader->curInvalid = 0; return 0; } int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) { SWal *pWal = pReader->pWal; - if (!pReader->curInvalid && ver == pReader->curVersion) { + if (ver == pReader->curVersion) { wDebug("vgId:%d, wal index:%" PRId64 " match, no need to reset", pReader->pWal->cfg.vgId, ver); return 0; } @@ -238,7 +234,7 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { wDebug("vgId:%d, wal starts to fetch head, index:%" PRId64, pRead->pWal->cfg.vgId, fetchVer); - if (pRead->curInvalid || pRead->curVersion != fetchVer) { + if (pRead->curVersion != fetchVer) { if (walReadSeekVer(pRead, fetchVer) < 0) { // pRead->curVersion = fetchVer; // pRead->curInvalid = 1; @@ -344,7 +340,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { return -1; } - if (pRead->curInvalid || pRead->curVersion != ver) { + if (pRead->curVersion != ver) { code = walReadSeekVer(pRead, ver); if (code < 0) { // pRead->curVersion = ver; @@ -479,7 +475,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { taosThreadMutexLock(&pReader->mutex); - if (pReader->curInvalid || pReader->curVersion != ver) { + if (pReader->curVersion != ver) { if (walReadSeekVer(pReader, ver) < 0) { wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); @@ -575,7 +571,6 @@ void walReadReset(SWalReader *pReader) { taosThreadMutexLock(&pReader->mutex); taosCloseFile(&pReader->pIdxFile); taosCloseFile(&pReader->pLogFile); - pReader->curInvalid = 1; pReader->curFileFirstVer = -1; pReader->curVersion = -1; taosThreadMutexUnlock(&pReader->mutex); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index cfaeca8038..0c3da80bff 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -627,6 +627,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_INVALID_FILE, "Index file is inval TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to see log") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") From 55fb48b6993bddaee4dd10df42c2bd8ac61e74f8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 18 Mar 2023 18:48:50 +0800 Subject: [PATCH 010/176] fix:error in TD-23218 & remove useless logic --- include/common/tcommon.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 5c2089dfde..ce242be7f5 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -209,7 +209,6 @@ typedef struct SSDataBlock { enum { FETCH_TYPE__DATA = 1, - FETCH_TYPE__SEP, FETCH_TYPE__NONE, }; From a5144f0ba214893cffb76d3994809f5cee5690be Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 20 Mar 2023 10:10:07 +0800 Subject: [PATCH 011/176] fix:set offset if read wal none --- include/common/tcommon.h | 2 +- source/dnode/vnode/src/tq/tq.c | 11 ----------- source/libs/executor/src/scanoperator.c | 3 ++- source/libs/parser/src/parInsertUtil.c | 6 +++--- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index ce242be7f5..691aae5593 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -208,7 +208,7 @@ typedef struct SSDataBlock { } SSDataBlock; enum { - FETCH_TYPE__DATA = 1, + FETCH_TYPE__DATA = 0, FETCH_TYPE__NONE, }; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ba18cb4ff8..8082e74d34 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -336,17 +336,6 @@ static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t su } pRsp->withTbName = 0; -#if 0 - pRsp->withTbName = pReq->withTbName; - if (pRsp->withTbName) { - pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); - if (pRsp->blockTbName == NULL) { - // TODO free - return -1; - } - } -#endif - pRsp->withSchema = false; return 0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2989410b2e..f467921937 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1623,13 +1623,14 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { while (1) { SFetchRet ret = {0}; tqNextBlock(pInfo->tqReader, &ret); - pTaskInfo->streamInfo.currentOffset = ret.offset; if (ret.fetchType == FETCH_TYPE__DATA) { blockDataCleanup(pInfo->pRes); setBlockIntoRes(pInfo, &ret.data, true); qDebug("queue scan log return %d rows", pInfo->pRes->info.rows); return pInfo->pRes; + }else{ + pTaskInfo->streamInfo.currentOffset = ret.offset; } return NULL; } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 52d3569dcd..66977f206a 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -469,11 +469,11 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { if (TSDB_CODE_SUCCESS == code) { SVgroupDataCxt* pVgCxt = NULL; int32_t vgId = pTableCxt->pMeta->vgId; - void** p = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); - if (NULL == p) { + void** pp = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); + if (NULL == pp) { code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); } else { - pVgCxt = *(SVgroupDataCxt**)p; + pVgCxt = *(SVgroupDataCxt**)pp; } if (TSDB_CODE_SUCCESS == code) { code = fillVgroupDataCxt(pTableCxt, pVgCxt); From c0b41ea0b3c9d0a1a4b10abb4e5901dc529798d2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 20 Mar 2023 14:09:13 +0800 Subject: [PATCH 012/176] fix:error in TD-23218 & remove useless logic --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/meta/metaSnapshot.c | 4 +- source/dnode/vnode/src/tq/tqExec.c | 3 + source/libs/executor/src/scanoperator.c | 5 +- utils/test/c/tmq_taosx_ci.c | 558 ++++++++++----------- 5 files changed, 287 insertions(+), 285 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 69fba6b651..2a720eb589 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -227,7 +227,7 @@ typedef struct SSnapContext { SArray *idList; int32_t index; bool withMeta; - bool queryMetaOrData; // true-get meta, false-get data + bool queryMeta; // true-get meta, false-get data } SSnapContext; typedef struct STqReader { diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 67ade45732..0126d29cc9 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -268,7 +268,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t ctx->snapVersion = snapVersion; ctx->suid = suid; ctx->subType = subType; - ctx->queryMetaOrData = withMeta; + ctx->queryMeta = withMeta; ctx->withMeta = withMeta; ctx->idVersion = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (ctx->idVersion == NULL) { @@ -475,7 +475,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in if (ctx->index >= taosArrayGetSize(ctx->idList)) { metaDebug("tmqsnap get meta end"); ctx->index = 0; - ctx->queryMetaOrData = false; // change to get data + ctx->queryMeta = false; // change to get data return 0; } diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index c0f1fad32c..b5434e0b01 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -169,6 +169,9 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta SMqMetaRsp* tmp = qStreamExtractMetaMsg(task); if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { + *pOffset = tmp->rspOffset; + qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); + tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META; tqDebug("tmqsnap task exec change to get data"); continue; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index f467921937..e29d8806e4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2109,9 +2109,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return NULL; } - if (!sContext->queryMetaOrData) { // change to get data next poll request - tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, 0, INT64_MIN); - pTaskInfo->streamInfo.metaRsp.rspOffset = pTaskInfo->streamInfo.currentOffset; + if (!sContext->queryMeta) { // change to get data next poll request + tqOffsetResetToData(&pTaskInfo->streamInfo.metaRsp.rspOffset, 0, INT64_MIN); } else { tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid); pTaskInfo->streamInfo.metaRsp.rspOffset = pTaskInfo->streamInfo.currentOffset; diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 1f25eae366..5baec3b4fd 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -101,285 +101,285 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { taos_free_result(pRes); /* test for TD-20612 end*/ - - pRes = taos_query(pConn, - "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct0 values(1626006833400, 1, 2, 'a')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query( - pConn, - "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, " - "'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, - "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) " - "(1626006833609, 51, 62, 'c333', 940)"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct3 select * from ct1"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); - if (taos_errno(pRes) != 0) { - printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - if (g_conf.dropTable) { - pRes = taos_query(pConn, "drop table ct3, ct1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - } - - pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); - if (taos_errno(pRes) != 0) { - printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 comment 'hello'"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 drop column c1"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into n1 values(now, 'eeee', 8989898899999) (now+9s, 'c333', 940)"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - if (g_conf.dropTable) { - pRes = taos_query(pConn, "drop table n1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - } - - pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt2 using jt tags('')"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into jt1 values(now, 1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into jt2 values(now, 11)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - if (g_conf.dropTable) { - pRes = taos_query(pConn, - "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - } - - pRes = taos_query(pConn, - "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, - "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query( - pConn, - "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) " - "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = - taos_query(pConn, - "insert into stt1 values(now + 2s, 3, 2, 'stt1') stt3 using stt tags(23, \"stt3\", true) values(now + " - "1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') " - "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb " - "tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); +// +// pRes = taos_query(pConn, +// "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " +// "nchar(8), t4 bool)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into ct0 values(1626006833400, 1, 2, 'a')"); +// if (taos_errno(pRes) != 0) { +// printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); +// if (taos_errno(pRes) != 0) { +// printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query( +// pConn, +// "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, " +// "'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); +// if (taos_errno(pRes) != 0) { +// printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); +// if (taos_errno(pRes) != 0) { +// printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, +// "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) " +// "(1626006833609, 51, 62, 'c333', 940)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into ct3 select * from ct1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); +// if (taos_errno(pRes) != 0) { +// printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); +// if (taos_errno(pRes) != 0) { +// printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// if (g_conf.dropTable) { +// pRes = taos_query(pConn, "drop table ct3, ct1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop table st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// } +// +// pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); +// if (taos_errno(pRes) != 0) { +// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); +// if (taos_errno(pRes) != 0) { +// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "alter table n1 comment 'hello'"); +// if (taos_errno(pRes) != 0) { +// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "alter table n1 drop column c1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into n1 values(now, 'eeee', 8989898899999) (now+9s, 'c333', 940)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to insert into n1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// if (g_conf.dropTable) { +// pRes = taos_query(pConn, "drop table n1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// } +// +// pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table jt2 using jt tags('')"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into jt1 values(now, 1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table jt1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into jt2 values(now, 11)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// if (g_conf.dropTable) { +// pRes = taos_query(pConn, +// "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " +// "nchar(8), t4 bool)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop table st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// } +// +// pRes = taos_query(pConn, +// "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " +// "nchar(8), t4 bool)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, +// "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " +// "nchar(8), t4 bool)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = taos_query( +// pConn, +// "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) " +// "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); +// +// pRes = +// taos_query(pConn, +// "insert into stt1 values(now + 2s, 3, 2, 'stt1') stt3 using stt tags(23, \"stt3\", true) values(now + " +// "1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') " +// "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb " +// "tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); +// return -1; +// } +// taos_free_result(pRes); return 0; } From 4228c24f2e26cc50ca9f4c07e8c8319c140290a6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 20 Mar 2023 14:12:37 +0800 Subject: [PATCH 013/176] fix:error in TD-23218 & remove useless logic --- utils/test/c/tmq_taosx_ci.c | 558 ++++++++++++++++++------------------ 1 file changed, 279 insertions(+), 279 deletions(-) diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 5baec3b4fd..1f25eae366 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -101,285 +101,285 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { taos_free_result(pRes); /* test for TD-20612 end*/ -// -// pRes = taos_query(pConn, -// "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " -// "nchar(8), t4 bool)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "insert into ct0 values(1626006833400, 1, 2, 'a')"); -// if (taos_errno(pRes) != 0) { -// printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); -// if (taos_errno(pRes) != 0) { -// printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query( -// pConn, -// "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, " -// "'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); -// if (taos_errno(pRes) != 0) { -// printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); -// if (taos_errno(pRes) != 0) { -// printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, -// "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) " -// "(1626006833609, 51, 62, 'c333', 940)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "insert into ct3 select * from ct1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); -// if (taos_errno(pRes) != 0) { -// printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); -// if (taos_errno(pRes) != 0) { -// printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// if (g_conf.dropTable) { -// pRes = taos_query(pConn, "drop table ct3, ct1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "drop table st1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// } -// -// pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); -// if (taos_errno(pRes) != 0) { -// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); -// if (taos_errno(pRes) != 0) { -// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "alter table n1 comment 'hello'"); -// if (taos_errno(pRes) != 0) { -// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "alter table n1 drop column c1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "insert into n1 values(now, 'eeee', 8989898899999) (now+9s, 'c333', 940)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to insert into n1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// if (g_conf.dropTable) { -// pRes = taos_query(pConn, "drop table n1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// } -// -// pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table jt2 using jt tags('')"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "insert into jt1 values(now, 1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table jt1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "insert into jt2 values(now, 11)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// if (g_conf.dropTable) { -// pRes = taos_query(pConn, -// "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " -// "nchar(8), t4 bool)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "drop table st1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// } -// -// pRes = taos_query(pConn, -// "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " -// "nchar(8), t4 bool)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, -// "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " -// "nchar(8), t4 bool)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query( -// pConn, -// "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) " -// "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = -// taos_query(pConn, -// "insert into stt1 values(now + 2s, 3, 2, 'stt1') stt3 using stt tags(23, \"stt3\", true) values(now + " -// "1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') " -// "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb " -// "tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); + + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct0 values(1626006833400, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query( + pConn, + "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, " + "'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, + "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) " + "(1626006833609, 51, 62, 'c333', 940)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 select * from ct1"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); + if (taos_errno(pRes) != 0) { + printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if (g_conf.dropTable) { + pRes = taos_query(pConn, "drop table ct3, ct1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop table st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); + if (taos_errno(pRes) != 0) { + printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 comment 'hello'"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 drop column c1"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into n1 values(now, 'eeee', 8989898899999) (now+9s, 'c333', 940)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if (g_conf.dropTable) { + pRes = taos_query(pConn, "drop table n1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table jt2 using jt tags('')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into jt1 values(now, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into jt2 values(now, 11)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if (g_conf.dropTable) { + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop table st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + pRes = taos_query(pConn, + "create stable if not exists stt (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table stt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, + "create stable if not exists sttb (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table sttb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query( + pConn, + "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) " + "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = + taos_query(pConn, + "insert into stt1 values(now + 2s, 3, 2, 'stt1') stt3 using stt tags(23, \"stt3\", true) values(now + " + "1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') " + "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb " + "tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); return 0; } From f94b1df529dc66c6ee0d3ad4cd30c319818fddbb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 20 Mar 2023 14:55:11 +0800 Subject: [PATCH 014/176] fix:error in TD-23218 & remove useless logic --- source/libs/executor/src/scanoperator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e29d8806e4..e4e7f32c36 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2091,7 +2091,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { qDebug("tmqsnap read snapshot done, change to get data from wal"); tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->sContext->snapVersion); } else { - tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, mtInfo.uid, INT64_MIN); + STqOffsetVal offset = {0}; + tqOffsetResetToData(&offset, mtInfo.uid, INT64_MIN); + qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); } tDeleteSSchemaWrapper(mtInfo.schema); From f80078030c448bf93cd711953842c034d88bc8bc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 20 Mar 2023 15:40:01 +0800 Subject: [PATCH 015/176] fix:error in TD-23218 & remove useless logic --- include/libs/executor/executor.h | 2 ++ source/dnode/vnode/src/tq/tq.c | 4 ++-- source/dnode/vnode/src/tq/tqExec.c | 3 +++ source/libs/executor/src/executor.c | 5 +++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index dcc3c86171..66ad0714bd 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -198,6 +198,8 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); +int64_t qStreamExtractOffsetUid(qTaskInfo_t tinfo); + SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8082e74d34..653eb2b9c4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -486,9 +486,9 @@ static int32_t processSubDbOrTable(STQ* pTq, STqHandle* pHandle, const SMqPollRe } tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 - ",version:%" PRId64, + ",ts:%" PRId64, pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid, - taosxRsp.rspOffset.version); + taosxRsp.rspOffset.ts); if (taosxRsp.blockNum > 0) { code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); tDeleteSTaosxRsp(&taosxRsp); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index b5434e0b01..d8e35f966f 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -157,6 +157,9 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (qStreamExtractOffsetUid(task) != 0) { + continue; + } tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); break; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fa576329a6..02cda670f0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -993,6 +993,11 @@ SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { return &pTaskInfo->streamInfo.metaRsp; } +int64_t qStreamExtractOffsetUid(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + return pTaskInfo->streamInfo.currentOffset.uid; +} + void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; memcpy(pOffset, &pTaskInfo->streamInfo.currentOffset, sizeof(STqOffsetVal)); From 506c192bff9071e179f6212a63fae58a691b139a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 20 Mar 2023 16:26:15 +0800 Subject: [PATCH 016/176] fix:error in TD-23218 & remove useless logic --- include/libs/executor/executor.h | 2 -- source/dnode/vnode/src/tq/tqExec.c | 35 +++++++++++-------------- source/libs/executor/src/executor.c | 5 ---- source/libs/executor/src/scanoperator.c | 12 ++++----- 4 files changed, 22 insertions(+), 32 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 66ad0714bd..dcc3c86171 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -198,8 +198,6 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); -int64_t qStreamExtractOffsetUid(qTaskInfo_t tinfo); - SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index d8e35f966f..19c8f8af3e 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -156,36 +156,33 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } } - if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - if (qStreamExtractOffsetUid(task) != 0) { + // get meta + SMqMetaRsp* tmp = qStreamExtractMetaMsg(task); + if (tmp->metaRspLen > 0) { + qStreamExtractOffset(task, &tmp->rspOffset); + *pMetaRsp = *tmp; + + tqDebug("tmqsnap task get data"); + break; + } + + if (pDataBlock == NULL) { + qStreamExtractOffset(task, pOffset); + if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { continue; } - tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), - pHandle->snapshotVer + 1); + tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); + qStreamExtractOffset(task, &pRsp->rspOffset); break; } if (pRsp->blockNum > 0) { tqDebug("tmqsnap task exec exited, get data"); + qStreamExtractOffset(task, &pRsp->rspOffset); break; } - - SMqMetaRsp* tmp = qStreamExtractMetaMsg(task); - if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { - *pOffset = tmp->rspOffset; - qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); - tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META; - tqDebug("tmqsnap task exec change to get data"); - continue; - } - - *pMetaRsp = *tmp; - tqDebug("tmqsnap task exec exited, get meta"); - break; } - qStreamExtractOffset(task, &pRsp->rspOffset); - return 0; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 02cda670f0..fa576329a6 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -993,11 +993,6 @@ SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { return &pTaskInfo->streamInfo.metaRsp; } -int64_t qStreamExtractOffsetUid(qTaskInfo_t tinfo) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return pTaskInfo->streamInfo.currentOffset.uid; -} - void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; memcpy(pOffset, &pTaskInfo->streamInfo.currentOffset, sizeof(STqOffsetVal)); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e4e7f32c36..45ee6e9ea1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2087,17 +2087,16 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { } SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); + STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->sContext->snapVersion); + tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion); } else { - STqOffsetVal offset = {0}; tqOffsetResetToData(&offset, mtInfo.uid, INT64_MIN); - qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); } + qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); tDeleteSSchemaWrapper(mtInfo.schema); - qDebug("tmqsnap stream scan tsdb return null"); return NULL; } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) { SSnapContext* sContext = pInfo->sContext; @@ -2112,10 +2111,11 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { } if (!sContext->queryMeta) { // change to get data next poll request - tqOffsetResetToData(&pTaskInfo->streamInfo.metaRsp.rspOffset, 0, INT64_MIN); + STqOffsetVal offset = {0}; + tqOffsetResetToData(&offset, 0, INT64_MIN); + qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); } else { tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid); - pTaskInfo->streamInfo.metaRsp.rspOffset = pTaskInfo->streamInfo.currentOffset; pTaskInfo->streamInfo.metaRsp.resMsgType = type; pTaskInfo->streamInfo.metaRsp.metaRspLen = dataLen; pTaskInfo->streamInfo.metaRsp.metaRsp = data; From c07e563ecd1f1524d4c0d8a1d6eb69a1c429d073 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 22 Mar 2023 10:43:11 +0800 Subject: [PATCH 017/176] fix:error in optimize consume logic --- source/dnode/vnode/src/tq/tqExec.c | 15 ++++++--------- source/dnode/vnode/src/tq/tqRead.c | 4 ++-- source/libs/executor/src/projectoperator.c | 8 ++------ source/libs/executor/src/scanoperator.c | 10 ++++++---- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 3063f0d372..0dbec85e6e 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -76,13 +76,13 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; - tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId); + tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task start execute", pHandle->consumerId, pTq->pVnode->config.vgId); if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + tqError("consumer:0x%"PRIx64" vgId:%d, task exec error since %s", pHandle->consumerId, pTq->pVnode->config.vgId, terrstr()); return -1; } - tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task executed, get %p", pHandle->consumerId, pTq->pVnode->config.vgId, pDataBlock); + tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task end execute, get block:%p", pHandle->consumerId, pTq->pVnode->config.vgId, pDataBlock); // current scan should be stopped asap, since the rebalance occurs. if (pDataBlock == NULL) { @@ -91,12 +91,9 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); pRsp->blockNum++; - - if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - rowCnt += pDataBlock->info.rows; - if (rowCnt >= 4096) { - break; - } + rowCnt += pDataBlock->info.rows; + if (rowCnt >= 4096) { + break; } } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 5260a00915..9579eb3407 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -309,7 +309,7 @@ void tqNextBlock(STqReader* pReader, SFetchRet* ret) { ret->offset.type = TMQ_OFFSET__LOG; ret->offset.version = pReader->pWalReader->curVersion; ret->fetchType = FETCH_TYPE__NONE; - tqInfo("return offset %" PRId64 ", no more valid msg in wal", ret->offset.version); + tqInfo("wal return none, offset %" PRId64 ", no more valid msg in wal", ret->offset.version); return; } @@ -327,7 +327,7 @@ void tqNextBlock(STqReader* pReader, SFetchRet* ret) { continue; } ret->fetchType = FETCH_TYPE__DATA; - tqDebug("return data rows %d", ret->data.info.rows); + tqDebug("wal return data rows %d", ret->data.info.rows); return; } } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 9fff7a4943..fdce9a95df 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -256,13 +256,9 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { if (pBlock == NULL) { if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE && pFinalRes->info.rows == 0) { pOperator->status = OP_OPENED; - if (pOperator->status == OP_EXEC_RECV) { - continue; - } else { - return NULL; - } + return NULL; } - qDebug("set op close, exec %d, status %d rows %d", pTaskInfo->execModel, pOperator->status, + qDebug("set op close, exec mode:%d, status %d rows %d", pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); setOperatorCompleted(pOperator); break; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 45ee6e9ea1..4e6cd148d1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1627,12 +1627,14 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (ret.fetchType == FETCH_TYPE__DATA) { blockDataCleanup(pInfo->pRes); setBlockIntoRes(pInfo, &ret.data, true); - qDebug("queue scan log return %d rows", pInfo->pRes->info.rows); - return pInfo->pRes; - }else{ + if (pInfo->pRes->info.rows > 0) { + qDebug("queue scan log return %d rows", pInfo->pRes->info.rows); + return pInfo->pRes; + } + }else if(ret.fetchType == FETCH_TYPE__NONE){ pTaskInfo->streamInfo.currentOffset = ret.offset; + return NULL; } - return NULL; } } else { qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.currentOffset.type); From 6fae38619d201cbaec882dbf07b80e62a0722b1b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 22 Mar 2023 19:36:12 +0800 Subject: [PATCH 018/176] fix:error in TD-23218 & remove useless logic --- source/dnode/vnode/src/tq/tq.c | 9 --------- source/dnode/vnode/src/tq/tqRead.c | 5 ++--- source/libs/executor/src/scanoperator.c | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8e3f1aa82a..610db77741 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -453,15 +453,6 @@ static int32_t processSubColumn(STQ* pTq, STqHandle* pHandle, const SMqPollReq* dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); taosWUnLockLatch(&pTq->pushLock); - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); - - // NOTE: this pHandle->consumerId may have been changed already. - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64 - ", ts:%" PRId64", reqId:0x%"PRIx64, - pRequest->consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid, - dataRsp.rspOffset.ts, pRequest->reqId); - - tDeleteSMqDataRsp(&dataRsp); return code; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index b002140a6c..6d4a8a1c76 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -357,11 +357,10 @@ bool tqNextDataBlock2(STqReader* pReader) { return false; } - tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, - pReader->msg2.ver, pReader->nextBlk); - int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, + pReader->msg2.ver, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pReader->tbIdHash == NULL) return true; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef0f391709..31f022f368 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1628,7 +1628,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { while (1) { SFetchRet ret = {0}; tqNextBlock(pInfo->tqReader, &ret); - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion); + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); if (ret.fetchType == FETCH_TYPE__DATA) { qDebug("doQueueScan get data from log %d rows, version:%" PRId64, pInfo->pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); From 8bf49fdbd2f9fa0a60f3c532f3d049f22bfc19d3 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 24 Mar 2023 17:16:06 +0800 Subject: [PATCH 019/176] fix: skiplist level --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d34af9acae..d8253e25f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -16,7 +16,7 @@ #include "tsdb.h" #define MEM_MIN_HASH 1024 -#define SL_MAX_LEVEL 5 +#define SL_MAX_LEVEL 1 // sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l) * 2 #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4)) From 8a89550f16f48d47a39c6e6605909b2f7ddb129a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 25 Mar 2023 13:10:14 +0800 Subject: [PATCH 020/176] fix: skiplist level --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- source/dnode/vnode/src/vnd/vnodeCfg.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d8253e25f7..d34af9acae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -16,7 +16,7 @@ #include "tsdb.h" #define MEM_MIN_HASH 1024 -#define SL_MAX_LEVEL 1 +#define SL_MAX_LEVEL 5 // sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l) * 2 #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4)) diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index c326c8bfac..54d2eabfa4 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -29,7 +29,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI, .update = 1, .compression = 2, - .slLevel = 5, + .slLevel = 1, .days = 14400, .minRows = 100, .maxRows = 4096, From 750cd8c5a4e993fab974a6eee05f047b7f5160ed Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 25 Mar 2023 16:33:47 +0800 Subject: [PATCH 021/176] fix:ci error --- source/dnode/vnode/src/tq/tq.c | 6 +++++- tests/system-test/7-tmq/subscribeStb0.py | 14 +++++++------- tests/system-test/7-tmq/subscribeStb2.py | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 610db77741..a58c9530ba 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -500,8 +500,12 @@ static int32_t processSubDbOrTable(STQ* pTq, STqHandle* pHandle, const SMqPollRe code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); tDeleteSTaosxRsp(&taosxRsp); return code; + }else { + *offset = taosxRsp.rspOffset; } - } else { + } + + if (offset->type == TMQ_OFFSET__LOG){ int64_t fetchVer = offset->version + 1; pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { diff --git a/tests/system-test/7-tmq/subscribeStb0.py b/tests/system-test/7-tmq/subscribeStb0.py index 1463cad627..bac1646457 100644 --- a/tests/system-test/7-tmq/subscribeStb0.py +++ b/tests/system-test/7-tmq/subscribeStb0.py @@ -228,7 +228,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -303,7 +303,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -315,7 +315,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -333,7 +333,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt: + if totalConsumeRows < expectrowcnt: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") @@ -386,7 +386,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -398,7 +398,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -416,7 +416,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != (expectrowcnt * (1 + 1/4)): + if totalConsumeRows < (expectrowcnt * (1 + 1/4)): tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index 6c3e122902..ae771d0556 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -233,7 +233,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) From 3a12e36b4b8f19180cbffcd5b3ab95547b6865e0 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 27 Mar 2023 14:13:17 +0800 Subject: [PATCH 022/176] fix:add combine function for last row --- source/libs/function/src/builtins.c | 7 ++++--- tests/script/tsim/stream/distributeInterval0.sim | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4fcc9ac1ad..7924f16ee9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2500,7 +2500,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = firstLastScalarFunction, .pPartialFunc = "_last_row_partial", .pMergeFunc = "_last_row_merge", - .finalizeFunc = firstLastFinalize + .finalizeFunc = firstLastFinalize, + .combineFunc = lastCombine }, { .name = "_cache_last_row", @@ -2809,7 +2810,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mode", .type = FUNCTION_TYPE_MODE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateMode, .getEnvFunc = getModeFuncEnv, .initFunc = modeFunctionSetup, @@ -3212,7 +3213,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_block_dist", .type = FUNCTION_TYPE_BLOCK_DIST, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateBlockDistFunc, .getEnvFunc = getBlockDistFuncEnv, .initFunc = blockDistSetup, diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index de1c72f8dc..1559d3d32b 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -284,7 +284,7 @@ sql create table ccc using st tags(3,2,2); sql create table ddd using st tags(4,2,2); -sql create stream streams1 ignore expired 0 fill_history 0 watermark 3s into streamst subtable(c) as select _wstart, c , count(*) c1 from st partition by c interval(1s) ; +sql create stream streams1 ignore expired 0 fill_history 0 watermark 3s into streamst subtable(c) as select _wstart, c , count(*) c1, last_row(b) c2 from st partition by c interval(1s) ; sql insert into aaa values(1648791221001,2,2,"/a1/aa/aa"); sql insert into bbb values(1648791221001,2,2,"/a1/aa/aa"); From cf2d169f751e60ffbf26824859a58d4fe985d3ba Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 27 Mar 2023 16:15:15 +0800 Subject: [PATCH 023/176] feat:coverity scan --- include/libs/stream/tstream.h | 2 +- source/libs/stream/src/stream.c | 8 ++--- .../stream/distributeIntervalRetrive0.sim | 31 ++++++++++++++++--- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 1d301623b1..dc374fa447 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -587,7 +587,7 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); -int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); +// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); int32_t streamTryExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 60729c4d0e..a4a02b7d65 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -275,7 +275,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S return 0; } -int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) { - // - return 0; -} +// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) { +// // +// return 0; +// } diff --git a/tests/script/tsim/stream/distributeIntervalRetrive0.sim b/tests/script/tsim/stream/distributeIntervalRetrive0.sim index 529a2a1b30..a2d9c4ab45 100644 --- a/tests/script/tsim/stream/distributeIntervalRetrive0.sim +++ b/tests/script/tsim/stream/distributeIntervalRetrive0.sim @@ -3,7 +3,6 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/exec.sh -n dnode1 -s start -#==system sh/exec.sh -n dnode1 -s start -v sleep 50 sql connect @@ -11,7 +10,6 @@ sql connect sql create dnode $hostname2 port 7200 system sh/exec.sh -n dnode2 -s start -#==system sh/exec.sh -n dnode2 -s start -v print ===== step1 $x = 0 @@ -37,14 +35,14 @@ endi print ===== step2 -sql create database test vgroups 4; +sql create database test vgroups 10; sql use test; sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 into streamtST1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); +sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 delete_mark 10s into streamtST1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); sleep 1000 @@ -235,6 +233,29 @@ endi print loop3 over +sql insert into ts1 values(1648791200001,1,12,3,1.0); +sql insert into ts2 values(1648791200001,1,12,3,1.0); +sql insert into ts3 values(1648791200001,1,12,3,1.0); +sql insert into ts4 values(1648791200001,1,12,3,1.0); + +$loop_count = 0 +loop31: +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamtST1; + +if $rows <= 4 then + print =====rows=$rows + goto loop31 +endi + +print loop31 over + sql drop stream if exists streams1; sql drop database if exists test1; @@ -243,7 +264,7 @@ sql use test1; sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ; +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 delete_mark 20s into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ; sql insert into t1 values(1648791211000,1,2,3); From 61571d92967c9da09aaf6ee1666aa83977ebb7e0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 27 Mar 2023 19:40:29 +0800 Subject: [PATCH 024/176] enh: optimize count performance --- source/common/src/tdatablock.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 209 +++++++++++++++++++++++- source/libs/executor/src/scanoperator.c | 9 +- 3 files changed, 213 insertions(+), 7 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3c8d394b43..a75046d06d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1180,7 +1180,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { void blockDataEmpty(SSDataBlock* pDataBlock) { SDataBlockInfo* pInfo = &pDataBlock->info; - if (pInfo->capacity == 0 || pInfo->rows > pDataBlock->info.capacity) { + if (pInfo->capacity == 0) { return; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 96bce02b67..c6444b5ded 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -24,6 +24,11 @@ typedef enum { EXTERNAL_ROWS_NEXT = 0x3, } EContentData; +typedef enum { + READ_MODE_COUNT_ONLY = 0x1, + READ_MODE_ALL, +} EReadMode; + typedef struct { STbDataIter* iter; int32_t index; @@ -167,6 +172,8 @@ struct STsdbReader { uint64_t suid; int16_t order; bool freeBlock; + EReadMode readMode; + uint64_t rowsNum; STimeWindow window; // the primary query time window that applies to all queries SSDataBlock* pResBlock; int32_t capacity; @@ -2998,6 +3005,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + return code; + } code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3006,12 +3016,19 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // build composed data block code = buildComposedDataBlock(pReader); } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + return code; + } // data in memory that are earlier than current file block // rows in buffer should be less than the file block in asc, greater than file block in desc int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + return code; + } + // only return the rows in last block int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); ASSERT(tsLast >= pBlock->maxKey.ts); @@ -3069,6 +3086,131 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } + +static int32_t doSumBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { + int64_t st = taosGetTimestampUs(); + LRUHandle* handle = NULL; + int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); + if (code != TSDB_CODE_SUCCESS || handle == NULL) { + goto _end; + } + + int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + + SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); + size_t num = taosArrayGetSize(aBlockIdx); + if (num == 0) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return TSDB_CODE_SUCCESS; + } + + SBlockIdx* pBlockIdx = NULL; + int32_t i = 0; + for (int32_t i = 0; i < num; ++i) { + pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); + if (pBlockIdx->suid != pReader->suid) { + continue; + } + + STableBlockScanInfo** p = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); + if (p == NULL || *p == NULL) { + continue; + } + + STableBlockScanInfo *pScanInfo = *p; + tMapDataReset(&pScanInfo->mapData); + tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); + + SDataBlk block = {0}; + for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { + tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block); + pReader->rowsNum += block.nRow; + } + } + +_end: + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return code; +} + + +static int32_t readRowsCountFromFile(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + while (1) { + bool hasNext = false; + int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); + if (code) { + return code; + } + + if (!hasNext) { // no data files on disk + break; + } + + code = doSumBlockRows(pReader, pReader->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pReader->status.loadFromFile = false; + + return code; +} + +static int32_t readRowsCountFromStt(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockLoadInfo* pBlockLoadInfo = NULL; + + for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file + pBlockLoadInfo = &pLastBlockReader->pInfo[i]; + + if (!pLastBlockReader->pInfo[i].sttBlockLoaded) { + pLastBlockReader->pInfo[i].sttBlockLoaded = true; + + code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); + if (code) { + return code; + } + } + + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + + if (size >= 1) { + SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); + SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); + + // all identical + if (pStart->suid == pEnd->suid) { + if (pStart->suid != pReader->suid) { + // no qualified stt block existed + taosArrayClear(pBlockLoadInfo->aSttBlk); + continue; + } + } else { + for (int32_t i = 0; i < size; ++i) { + SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + uint64_t s = p->suid; + if (s < pReader->suid) { + continue; + } + + if (s == pReader->suid) { + pReader->rowsNum += p->nRow; + } else if (s > pReader->suid) { + break; + } + } + } + } + } + + return code; +} + + static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; STableUidList* pUidList = &pStatus->uidList; @@ -3212,6 +3354,12 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { initBlockDumpInfo(pReader, pBlockIter); } else { if (pReader->status.pCurrentFileset->nSttF > 0) { + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + pReader->pResBlock->info.rows = pReader->rowsNum; + pReader->rowsNum = 0; + return TSDB_CODE_SUCCESS; + } + // data blocks in current file are exhausted, let's try the next file now SBlockData* pBlockData = &pReader->status.fileBlockData; if (pBlockData->uid != 0) { @@ -3226,7 +3374,17 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { code = initForFirstBlockInFile(pReader, pBlockIter); // error happens or all the data files are completely checked - if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + pReader->pResBlock->info.rows = pReader->rowsNum; + pReader->rowsNum = 0; + return TSDB_CODE_SUCCESS; + } + + if (pReader->status.loadFromFile == false) { return code; } @@ -3240,6 +3398,17 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } code = doBuildDataBlock(pReader); + if (READ_MODE_COUNT_ONLY == pReader->readMode) { + if (false == pReader->status.composedDataBlock && pDumpInfo->allDumped) { + pReader->rowsNum += pReader->pResBlock->info.rows; + pReader->pResBlock->info.rows = 0; + continue; + } else if (pReader->pResBlock->info.rows == 0 && pReader->rowsNum > 0) { + pReader->pResBlock->info.rows = pReader->rowsNum; + pReader->rowsNum = 0; + return TSDB_CODE_SUCCESS; + } + } } if (code != TSDB_CODE_SUCCESS) { @@ -3986,6 +4155,8 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; + } else if (READ_MODE_COUNT_ONLY == pReader->readMode) { + // DO NOTHING } else { code = initForFirstBlockInFile(pReader, pBlockIter); } @@ -4091,6 +4262,9 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pReader->suspended = true; + + pReader->readMode = READ_MODE_COUNT_ONLY; + tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -4394,6 +4568,35 @@ _err: return code; } +static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->pResBlock; + + while (1) { + if (pReader->status.loadFromFile == false) { + break; + } + + code = readRowsCountFromFile(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = readRowsCountFromStt(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + } + + pBlock->info.rows = pReader->rowsNum; + pBlock->info.id.uid = 0; + pBlock->info.dataLoad = 0; + + pReader->rowsNum = 0; + + return pBlock->info.rows > 0; +} + static bool doTsdbNextDataBlock(STsdbReader* pReader) { // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; @@ -4404,6 +4607,10 @@ static bool doTsdbNextDataBlock(STsdbReader* pReader) { return false; } + if (READ_MODE_COUNT_ONLY == pReader->readMode) { + return tsdbReadRowsCountOnly(pReader); + } + if (pStatus->loadFromFile) { int32_t code = buildBlockFromFiles(pReader); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5dff1abb97..2af13f083d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -654,9 +654,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - ASSERT(pBlock->info.id.uid != 0); - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); - + if (pBlock->info.id.uid) { + pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + } + uint32_t status = 0; int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { @@ -680,7 +681,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; - ASSERT(pBlock->info.id.uid != 0); return pBlock; } return NULL; @@ -797,7 +797,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { SSDataBlock* result = doGroupedTableScan(pOperator); if (result != NULL) { - ASSERT(result->info.id.uid != 0); return result; } From 80798dd934a3f8df7c5e62a965404a6f9e2e06c3 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 09:20:53 +0800 Subject: [PATCH 025/176] fix: job retry issue --- include/libs/qcom/query.h | 1 + source/common/src/tdatablock.c | 2 +- source/libs/qcom/src/queryUtil.c | 2 + source/libs/scheduler/inc/schInt.h | 30 +++++++++---- source/libs/scheduler/src/schFlowCtrl.c | 1 - source/libs/scheduler/src/schJob.c | 42 ++++++++++++++++-- source/libs/scheduler/src/schRemote.c | 42 +++++++++++------- source/libs/scheduler/src/schStatus.c | 3 ++ source/libs/scheduler/src/schTask.c | 59 ++++++++++--------------- source/libs/scheduler/src/scheduler.c | 2 +- 10 files changed, 119 insertions(+), 65 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index cb547ee6b3..b6ada5a0c7 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -33,6 +33,7 @@ typedef enum { JOB_TASK_STATUS_INIT, JOB_TASK_STATUS_EXEC, JOB_TASK_STATUS_PART_SUCC, + JOB_TASK_STATUS_FETCH, JOB_TASK_STATUS_SUCC, JOB_TASK_STATUS_FAIL, JOB_TASK_STATUS_DROP, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3c8d394b43..a75046d06d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1180,7 +1180,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { void blockDataEmpty(SSDataBlock* pDataBlock) { SDataBlockInfo* pInfo = &pDataBlock->info; - if (pInfo->capacity == 0 || pInfo->rows > pDataBlock->info.capacity) { + if (pInfo->capacity == 0) { return; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index c68a08682c..9d8c170003 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -194,6 +194,8 @@ char* jobTaskStatusStr(int32_t status) { return "EXECUTING"; case JOB_TASK_STATUS_PART_SUCC: return "PARTIAL_SUCCEED"; + case JOB_TASK_STATUS_FETCH: + return "FETCHING"; case JOB_TASK_STATUS_SUCC: return "SUCCEED"; case JOB_TASK_STATUS_FAIL: diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 85b952937f..b7f9272bdc 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -193,7 +193,7 @@ typedef struct SSchLevel { int32_t taskSucceed; int32_t taskNum; int32_t taskLaunchedNum; - int32_t taskDoneNum; + int32_t taskExecDoneNum; SArray *subTasks; // Element is SSchTask } SSchLevel; @@ -340,6 +340,9 @@ extern SSchedulerMgmt schMgmt; #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) #define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task)) +#define SCH_TASK_ALREADY_LAUNCHED(task) (SCH_GET_TASK_STATUS(task) >= JOB_TASK_STATUS_EXEC) +#define SCH_TASK_EXEC_DONE(task) (SCH_GET_TASK_STATUS(task) >= JOB_TASK_STATUS_PART_SUCC) + #define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL) #define SCH_SET_TASK_HANDLE(_task, _handle) ((_task)->handle = (_handle)) @@ -361,6 +364,7 @@ extern SSchedulerMgmt schMgmt; (SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) #define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) #define SCH_TASK_NEED_FETCH(_task) ((_task)->plan->subplanType != SUBPLAN_TYPE_MODIFY) +#define SCH_MULTI_LEVEL_LAUNCHED(_job) ((_job)->levelIdx != ((_job)->levelNum - 1)) #define SCH_SET_JOB_TYPE(_job, type) \ do { \ @@ -377,16 +381,24 @@ extern SSchedulerMgmt schMgmt; #define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job)) #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) -#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) \ - (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)) || (_task)->redirectCtx.inRedirect)) #define SCH_REDIRECT_MSGTYPE(_msgType) \ ((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || \ (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) -#define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) \ - (SCH_REDIRECT_MSGTYPE(_msgType) && \ - (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen)))) -#define SCH_NEED_RETRY(_msgType, _code) \ - ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) +#define SCH_LOW_LEVEL_NETWORK_ERR(_job, _task, _code) \ + (SCH_NETWORK_ERR(_code) && ((_task)->level->level == (_job)->levelIdx)) +#define SCH_TOP_LEVEL_NETWORK_ERR(_job, _task, _code) \ + (SCH_NETWORK_ERR(_code) && ((_task)->level->level > (_job)->levelIdx)) +#define SCH_TASK_RETRY_NETWORK_ERR(_task, _code) \ + (SCH_NETWORK_ERR(_code) && (_task)->redirectCtx.inRedirect) + +#define SCH_JOB_NEED_RETRY(_job, _task, _msgType, _code) \ + (SCH_REDIRECT_MSGTYPE(_msgType) && SCH_TOP_LEVEL_NETWORK_ERR(_job, _task, _code)) +#define SCH_TASKSET_NEED_RETRY(_job, _task, _msgType, _code) \ + (SCH_REDIRECT_MSGTYPE(_msgType) && \ + (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_LOW_LEVEL_NETWORK_ERR((_job), (_task), (_code)) || SCH_TASK_RETRY_NETWORK_ERR((_task), (_code)))) +#define SCH_TASK_NEED_RETRY(_msgType, _code) \ + ((SCH_REDIRECT_MSGTYPE(_msgType) && SCH_NETWORK_ERR(_code)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) + #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum) #define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) @@ -562,7 +574,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq); int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes); int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet); -int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode); +int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode); void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode); int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq); void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode); diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 5e4fe4b8a1..9cb95a6bbe 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -282,7 +282,6 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) { } int32_t code = schLaunchTasksInFlowCtrlListImpl(pJob, ctrl); - ; SCH_ERR_RET(code); return code; // to avoid compiler error diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 980a8ac6a1..b2c90dc67f 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -108,10 +108,18 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && - newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) { + newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC && + newStatus != JOB_TASK_STATUS_FETCH) { SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } + break; + case JOB_TASK_STATUS_FETCH: + if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && + newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) { + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); + } + break; case JOB_TASK_STATUS_SUCC: case JOB_TASK_STATUS_FAIL: @@ -550,9 +558,9 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { } SSchLevel *pLevel = pTask->level; - int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1); + int32_t doneNum = atomic_add_fetch_32(&pLevel->taskExecDoneNum, 1); if (doneNum == pLevel->taskNum) { - pJob->levelIdx--; + atomic_sub_fetch_32(&pJob->levelIdx, 1); pLevel = taosArrayGet(pJob->levels, pJob->levelIdx); for (int32_t i = 0; i < pLevel->taskNum; ++i) { @@ -562,6 +570,10 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { continue; } + if (SCH_TASK_ALREADY_LAUNCHED(pTask)) { + continue; + } + SCH_ERR_RET(schLaunchTask(pJob, pTask)); } } @@ -811,6 +823,30 @@ void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode) { } } +int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) { + if (pJob->status >= JOB_TASK_STATUS_PART_SUCC) { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->fetched) { + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_TASK_ELOG("already fetched while got error %s", tstrerror(rspCode)); + SCH_ERR_JRET(rspCode); + } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + + SCH_ERR_JRET(schChkResetJobRetry(pJob, rspCode)); + +} + bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync) { bool r = false; SCH_LOCK(SCH_READ, &pJob->opStatus.lock); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 6f4130fd9f..af206aa46e 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -36,7 +36,7 @@ int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_QW_MSG_ERROR); } - if (taskStatus != JOB_TASK_STATUS_PART_SUCC) { + if (taskStatus != JOB_TASK_STATUS_FETCH) { SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_QW_MSG_ERROR); @@ -137,25 +137,12 @@ int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp) { return TSDB_CODE_SUCCESS; } -// Note: no more task error processing, handled in function internal -int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { +int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; char *msg = pMsg->pData; int32_t msgSize = pMsg->len; int32_t msgType = pMsg->msgType; - bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode)); - if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId)); - } - - SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType)); - - int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1); - if (SCH_TASK_NEED_REDIRECT(pTask, reqType, rspCode, pMsg->len)) { - SCH_RET(schHandleRedirect(pJob, pTask, (SDataBuf *)pMsg, rspCode)); - } - pTask->redirectCtx.inRedirect = false; switch (msgType) { @@ -423,6 +410,31 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } + + +// Note: no more task error processing, handled in function internal +int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + int32_t msgType = pMsg->msgType; + + bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode)); + if (SCH_IS_QUERY_JOB(pJob)) { + SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId)); + } + + SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType)); + + int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1); + if (SCH_JOB_NEED_RETRY(pJob, pTask, reqType, rspCode)) { + SCH_RET(schHandleJobRetry()); + } else if (SCH_TASKSET_NEED_RETRY(pJob, pTask, reqType, rspCode)) { + SCH_RET(schHandleTaskSetRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode)); + } + + pTask->redirectCtx.inRedirect = false; + + SCH_RET(schProcessResponseMsg(pJob, pTask, execId, pMsg, rspCode)); +} int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index 4c16a81a05..9d0ad30e2a 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -34,6 +34,9 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) { case JOB_TASK_STATUS_PART_SUCC: SCH_ERR_JRET(schProcessOnJobPartialSuccess(pJob)); break; + case JOB_TASK_STATUS_FETCH: + SCH_ERR_JRET(schJobFetchRows(pJob)); + break; case JOB_TASK_STATUS_SUCC: break; case JOB_TASK_STATUS_FAIL: diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index bdab739327..a3194c7ff7 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -404,21 +404,7 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { - int32_t code = 0; - - SCH_TASK_DLOG("task will be redirected now, status:%s, code:%s", SCH_GET_TASK_STATUS_STR(pTask), tstrerror(rspCode)); - - if (NULL == pData) { - pTask->retryTimes = 0; - } - - if (!NO_RET_REDIRECT_ERROR(rspCode)) { - SCH_UPDATE_REDICT_CODE(pJob, rspCode); - } - - SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL, rspCode)); - +void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { pTask->waitRetry = true; schDropTaskOnExecNode(pJob, pTask); @@ -426,10 +412,27 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 schRemoveTaskFromExecList(pJob, pTask); schDeregisterTaskHb(pJob, pTask); atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); + if (SCH_TASK_EXEC_DONE(pTask)) { + atomic_sub_fetch_32(&pTask->level->taskExecDoneNum, 1); + } taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; pTask->lastMsgType = 0; memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); +} + +int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { + int32_t code = 0; + + SCH_TASK_DLOG("task will be redirected now, status:%s, code:%s", SCH_GET_TASK_STATUS_STR(pTask), tstrerror(rspCode)); + + if (!NO_RET_REDIRECT_ERROR(rspCode)) { + SCH_UPDATE_REDICT_CODE(pJob, rspCode); + } + + SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL, rspCode)); + + schResetTaskForRetry(pJob, pTask); if (SCH_IS_DATA_BIND_TASK(pTask)) { if (pData && pData->pEpSet) { @@ -445,12 +448,6 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } - if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { - if (JOB_TASK_STATUS_EXEC == SCH_GET_TASK_STATUS(pTask)) { - SCH_ERR_JRET(schLaunchTasksInFlowCtrlList(pJob, pTask)); - } - } - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); SCH_ERR_JRET(schDelayLaunchTask(pJob, pTask)); @@ -486,20 +483,10 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } -int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { +int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; - if (JOB_TASK_STATUS_PART_SUCC == pJob->status) { - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->fetched) { - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); - SCH_TASK_ELOG("already fetched while got error %s", tstrerror(rspCode)); - SCH_ERR_JRET(rspCode); - } - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); - - schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); - } + SCH_ERR_JRET(schChkResetJobRetry(pJob, rspCode)); if (SYNC_OTHER_LEADER_REDIRECT_ERROR(rspCode)) { if (NULL == pData->pEpSet) { @@ -510,6 +497,7 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 } code = schDoTaskRedirect(pJob, pTask, pData, rspCode); + taosMemoryFreeClear(pData->pData); taosMemoryFreeClear(pData->pEpSet); @@ -645,7 +633,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo return TSDB_CODE_SUCCESS; } - if (!SCH_NEED_RETRY(pTask->lastMsgType, errCode)) { + if (!SCH_TASK_NEED_RETRY(pTask->lastMsgType, errCode)) { *needRetry = false; SCH_TASK_DLOG("task no more retry cause of errCode, errCode:%x - %s", errCode, tstrerror(errCode)); return TSDB_CODE_SUCCESS; @@ -1067,7 +1055,6 @@ int32_t schLaunchTaskImpl(void *param) { SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } - // NOTE: race condition: the task should be put into the hash table before send msg to server if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXEC) { SCH_ERR_JRET(schPushTaskToExecList(pJob, pTask)); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXEC); @@ -1272,6 +1259,8 @@ int32_t schLaunchFetchTask(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } + SCH_SET_TASK_STATUS(pJob->fetchTask, JOB_TASK_STATUS_FETCH); + if (SCH_IS_LOCAL_EXEC_TASK(pJob, pJob->fetchTask)) { SCH_ERR_JRET(schExecLocalFetch(pJob, pJob->fetchTask)); } else { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 7cd5e957b6..2b46a4710e 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -91,7 +91,7 @@ int32_t schedulerFetchRows(int64_t jobId, SSchedulerReq *pReq) { SCH_ERR_JRET(schHandleOpBeginEvent(jobId, &pJob, SCH_OP_FETCH, pReq)); - SCH_ERR_JRET(schJobFetchRows(pJob)); + SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_FETCH, pReq)); _return: From 2198ae2060a856bdb32e305b2d85b8da296045cd Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 28 Mar 2023 10:49:49 +0800 Subject: [PATCH 026/176] fix(query): spread/elapsed using sma result for computation --- source/libs/function/src/builtins.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4fcc9ac1ad..84de55653c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2375,7 +2375,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "spread", .type = FUNCTION_TYPE_SPREAD, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, .translateFunc = translateSpread, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getSpreadFuncEnv, @@ -2417,7 +2417,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "elapsed", .type = FUNCTION_TYPE_ELAPSED, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, .dataRequiredFunc = statisDataRequired, .translateFunc = translateElapsed, .getEnvFunc = getElapsedFuncEnv, From c36c5253c59b83235b33d2b7e0fc39b92aa84039 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 13:31:40 +0800 Subject: [PATCH 027/176] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5ad9276c6c..8953e4b907 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3929,8 +3929,13 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n if (code) { return code; } - pReader->status.uidList.tableUidList = - (uint64_t*)taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); + + char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pReader->status.uidList.tableUidList = (uint64_t*)p1; } taosHashClear(pReader->status.pTableMap); From 4fd3064fd9bd97bc2cece0579d0ca9ae15d654ab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 14:43:37 +0800 Subject: [PATCH 028/176] refactor: do some internal refactor. --- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tq/tq.c | 15 +-- source/dnode/vnode/src/tq/tqExec.c | 19 ++- source/dnode/vnode/src/tq/tqPush.c | 180 +++++++++++++++----------- 4 files changed, 119 insertions(+), 97 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0fe7f9a773..8b01ba237f 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -193,7 +193,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); int tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type); -int tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer); +int tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer); int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ccd8b885a0..2d2c70c84a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -223,19 +223,6 @@ static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqData int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { SMqDataRsp* pRsp = pPushEntry->pDataRsp; - -#if 0 - A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); - - A(!pRsp->withSchema); - A(taosArrayGetSize(pRsp->blockSchema) == 0); - - if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - A(pRsp->rspOffset.version > pRsp->reqOffset.version); - } -#endif - SMqRspHead* pHeader = &pPushEntry->pDataRsp->head; doSendDataRsp(&pPushEntry->info, pRsp, pHeader->epoch, pHeader->consumerId, pHeader->mqMsgType); @@ -895,7 +882,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg atomic_store_32(&pHandle->epoch, -1); // remove if it has been register in the push manager, and return one empty block to consumer - tqRemovePushEntry(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); + tqUnregisterPushEntry(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index ce86a6757e..6845350346 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -62,6 +62,8 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in } int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { + const int32_t MAX_ROWS_TO_RETURN = 4096; + const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; @@ -82,7 +84,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs } } - int32_t rowCnt = 0; + int32_t totalRows = 0; + while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; @@ -94,9 +97,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs return -1; } - tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task executed, get %p", pHandle->consumerId, vgId, pDataBlock); - - // current scan should be stopped asap, since the rebalance occurs. + // current scan should be stopped ASAP, since the re-balance occurs. if (pDataBlock == NULL) { break; } @@ -104,9 +105,12 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); pRsp->blockNum++; + tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%d, total blocks:%d", vgId, pHandle->consumerId, + pDataBlock->info.rows, pRsp->blockNum); + if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - rowCnt += pDataBlock->info.rows; - if (rowCnt >= 4096) { + totalRows += pDataBlock->info.rows; + if (totalRows >= MAX_ROWS_TO_RETURN) { break; } } @@ -127,6 +131,9 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs return -1; } + tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%d, total blocks:%d, rows:%d", vgId, pHandle->consumerId, + pRsp->blockNum, totalRows); + return 0; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 797aeb3f04..e07ab06458 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -206,6 +206,87 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ } #endif +typedef struct { + void* pKey; + int64_t keyLen; +} SItem; + +static void recordPushedEntry(SArray* cachedKey, void* pIter); + +static void freeItem(void* param) { + SItem* p = (SItem*) param; + taosMemoryFree(p->pKey); +} + +static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + size_t numOfKeys = taosArrayGetSize(pCachedKeys); + + for (int32_t i = 0; i < numOfKeys; i++) { + SItem* pItem = taosArrayGet(pCachedKeys, i); + if (taosHashRemove(pTq->pPushMgr, pItem->pKey, pItem->keyLen) != 0) { + tqError("vgId:%d, tq push hash remove key error, key: %s", vgId, (char*) pItem->pKey); + } + } + + if (numOfKeys > 0) { + tqDebug("vgId:%d, pushed %d items and remain:%d", vgId, numOfKeys, (int32_t)taosHashGetSize(pTq->pPushMgr)); + } +} + +static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int64_t ver, int32_t vgId, char* pData, + int32_t dataLen, SArray* pCachedKey) { + STqPushEntry* pPushEntry = *(STqPushEntry**)pIter; + + SMqDataRsp* pRsp = pPushEntry->pDataRsp; + if (pRsp->reqOffset.version >= ver) { + tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId, + pRsp->reqOffset.version, ver); + return; + } + + qTaskInfo_t pTaskInfo = pExec->task; + + // prepare scan mem data + SPackedData submit = { + .msgStr = pData, + .msgLen = dataLen, + .ver = ver, + }; + + if(qStreamSetScanMemData(pTaskInfo, submit) != 0){ + return; + } + + // here start to scan submit block to extract the subscribed data + int32_t totalRows = 0; + + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts = 0; + if (qExecTask(pTaskInfo, &pDataBlock, &ts) < 0) { + tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr()); + } + + if (pDataBlock == NULL) { + break; + } + + tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + pRsp->blockNum++; + totalRows += pDataBlock->info.rows; + } + + tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d, rows:%d", vgId, pPushEntry->subKey, pRsp->blockNum, + totalRows); + + if (pRsp->blockNum > 0) { + tqOffsetResetToLog(&pRsp->rspOffset, ver); + tqPushDataRsp(pTq, pPushEntry); + recordPushedEntry(pCachedKey, pIter); + } +} + int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg)); int32_t len = msgLen - sizeof(SSubmitReq2Msg); @@ -220,24 +301,19 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d", vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush); - SArray* cachedKeys = taosArrayInit(0, sizeof(void*)); - SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t)); - - void* data = taosMemoryMalloc(len); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to copy data for stream since out of memory"); - taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree); - taosArrayDestroy(cachedKeyLens); - - // unlock + tqError("failed to copy data for stream since out of memory, vgId:%d, consumer:0x%"PRIx64, vgId); taosWUnLockLatch(&pTq->lock); return -1; } memcpy(data, pReq, len); - void* pIter = NULL; + SArray* cachedKey = taosArrayInit(0, sizeof(SItem)); + void* pIter = NULL; + while (1) { pIter = taosHashIterate(pTq->pPushMgr, pIter); if (pIter == NULL) { @@ -248,83 +324,28 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey)); if (pHandle == NULL) { - tqDebug("vgId:%d, cannot find handle %s", pTq->pVnode->config.vgId, pPushEntry->subKey); - continue; - } - - SMqDataRsp* pRsp = pPushEntry->pDataRsp; - if (pRsp->reqOffset.version >= ver) { - tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId, - pRsp->reqOffset.version, ver); + tqDebug("vgId:%d, failed to find handle %s in pushing data to consumer, ignore", pTq->pVnode->config.vgId, pPushEntry->subKey); continue; } STqExecHandle* pExec = &pHandle->execHandle; - qTaskInfo_t task = pExec->task; - - // prepare scan mem data - SPackedData submit = { - .msgStr = data, - .msgLen = len, - .ver = ver, - }; - if(qStreamSetScanMemData(task, submit) != 0){ - continue; - } - - // here start to scan submit block to extract the subscribed data - while (1) { - SSDataBlock* pDataBlock = NULL; - uint64_t ts = 0; - if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr()); - } - - if (pDataBlock == NULL) { - break; - } - - tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); - pRsp->blockNum++; - } - - tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d", vgId, pPushEntry->subKey, pRsp->blockNum); - if (pRsp->blockNum > 0) { - // set offset - tqOffsetResetToLog(&pRsp->rspOffset, ver); - - // remove from hash - size_t kLen; - void* key = taosHashGetKey(pIter, &kLen); - void* keyCopy = taosMemoryCalloc(1, kLen + 1); - memcpy(keyCopy, key, kLen); - - taosArrayPush(cachedKeys, &keyCopy); - taosArrayPush(cachedKeyLens, &kLen); - - tqPushDataRsp(pTq, pPushEntry); - } + doPushDataForEntry(pIter, pExec, pTq, ver, vgId, data, len, cachedKey); } - // delete entry - for (int32_t i = 0; i < taosArrayGetSize(cachedKeys); i++) { - void* key = taosArrayGetP(cachedKeys, i); - size_t kLen = *(size_t*)taosArrayGet(cachedKeyLens, i); - if (taosHashRemove(pTq->pPushMgr, key, kLen) != 0) { - tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key); - } - } - - taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree); - taosArrayDestroy(cachedKeyLens); + doRemovePushedEntry(cachedKey, pTq); + taosArrayDestroyEx(cachedKey, freeItem); taosMemoryFree(data); } + // unlock taosWUnLockLatch(&pTq->lock); } if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode)) { - if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; + if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) { + return 0; + } + if (msgType == TDMT_VND_SUBMIT) { void* data = taosMemoryMalloc(len); if (data == NULL) { @@ -351,6 +372,13 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) return 0; } +void recordPushedEntry(SArray* cachedKey, void* pIter) { + size_t kLen = 0; + void* key = taosHashGetKey(pIter, &kLen); + SItem item = {.pKey = strndup(key, kLen), .keyLen = kLen}; + taosArrayPush(cachedKey, &item); +} + int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type) { uint64_t consumerId = pRequest->consumerId; @@ -388,8 +416,8 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, return 0; } -int32_t tqRemovePushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) { - int32_t vgId = TD_VID(pTq->pVnode); +int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) { + int32_t vgId = TD_VID(pTq->pVnode); STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen); if (pEntry != NULL) { From 68f310dd486421e6c577307c383054e8daf92bcc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 15:05:10 +0800 Subject: [PATCH 029/176] refactor: do some internal refactor. --- include/libs/stream/streamState.h | 23 +++++---- include/libs/stream/tstream.h | 5 +- source/dnode/vnode/CMakeLists.txt | 2 +- source/dnode/vnode/src/tq/tq.c | 4 ++ source/dnode/vnode/src/tq/tqPush.c | 20 +++----- .../dnode/vnode/src/tq/{tqExec.c => tqScan.c} | 47 +++++++++++-------- 6 files changed, 53 insertions(+), 48 deletions(-) rename source/dnode/vnode/src/tq/{tqExec.c => tqScan.c} (93%) diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 6b09bf4899..fd5cec2931 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -23,20 +23,19 @@ extern "C" { #ifndef _STREAM_STATE_H_ #define _STREAM_STATE_H_ -typedef struct SStreamTask SStreamTask; - typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); typedef struct STdbState { - SStreamTask* pOwner; - TDB* db; - TTB* pStateDb; - TTB* pFuncStateDb; - TTB* pFillStateDb; // todo refactor - TTB* pSessionStateDb; - TTB* pParNameDb; - TTB* pParTagDb; - TXN* txn; + struct SStreamTask* pOwner; + + TDB* db; + TTB* pStateDb; + TTB* pFuncStateDb; + TTB* pFillStateDb; // todo refactor + TTB* pSessionStateDb; + TTB* pParNameDb; + TTB* pParTagDb; + TXN* txn; } STdbState; // incremental state storage @@ -45,7 +44,7 @@ typedef struct { int32_t number; } SStreamState; -SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); +SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); void streamStateClose(SStreamState* pState); int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 1d301623b1..d04e965c6f 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -295,7 +295,7 @@ typedef struct { SEpSet epSet; } SStreamChildEpInfo; -typedef struct SStreamTask { +struct SStreamTask { int64_t streamId; int32_t taskId; int32_t totalLevel; @@ -362,8 +362,7 @@ typedef struct SStreamTask { int64_t checkpointingId; int32_t checkpointAlignCnt; - -} SStreamTask; +}; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 8dc3f46ae3..9911752f8e 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -57,7 +57,7 @@ target_sources( # tq "src/tq/tq.c" - "src/tq/tqExec.c" + "src/tq/tqScan.c" "src/tq/tqMeta.c" "src/tq/tqRead.c" "src/tq/tqOffset.c" diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2d2c70c84a..e1149b48af 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -473,6 +473,10 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); code = tqScanData(pTq, pHandle, &dataRsp, pOffset); + if (code != TSDB_CODE_SUCCESS) { + taosWUnLockLatch(&pTq->lock); + return code; + } // till now, all data has been transferred to consumer, new data needs to push client once arrived. if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index e07ab06458..73d722dba5 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -220,7 +220,7 @@ static void freeItem(void* param) { static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); - size_t numOfKeys = taosArrayGetSize(pCachedKeys); + int32_t numOfKeys = (int32_t) taosArrayGetSize(pCachedKeys); for (int32_t i = 0; i < numOfKeys; i++) { SItem* pItem = taosArrayGet(pCachedKeys, i); @@ -248,13 +248,9 @@ static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int6 qTaskInfo_t pTaskInfo = pExec->task; // prepare scan mem data - SPackedData submit = { - .msgStr = pData, - .msgLen = dataLen, - .ver = ver, - }; + SPackedData submit = {.msgStr = pData, .msgLen = dataLen, .ver = ver}; - if(qStreamSetScanMemData(pTaskInfo, submit) != 0){ + if (qStreamSetScanMemData(pTaskInfo, submit) != 0) { return; } @@ -287,7 +283,7 @@ static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int6 } } -int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { +int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg)); int32_t len = msgLen - sizeof(SSubmitReq2Msg); int32_t vgId = TD_VID(pTq->pVnode); @@ -341,6 +337,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) taosWUnLockLatch(&pTq->lock); } + // push data for stream processing if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode)) { if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) { return 0; @@ -353,12 +350,9 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) tqError("failed to copy data for stream since out of memory"); return -1; } + memcpy(data, pReq, len); - SPackedData submit = { - .msgStr = data, - .msgLen = len, - .ver = ver, - }; + SPackedData submit = {.msgStr = data, .msgLen = len, .ver = ver}; tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); tqProcessSubmitReq(pTq, submit); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqScan.c similarity index 93% rename from source/dnode/vnode/src/tq/tqExec.c rename to source/dnode/vnode/src/tq/tqScan.c index 6845350346..36d0631195 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -18,7 +18,9 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); - if (buf == NULL) return -1; + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; pRetrieve->useconds = 0; @@ -31,7 +33,8 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t actualLen += sizeof(SRetrieveTableRsp); taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockData, &buf); - return 0; + + return TSDB_CODE_SUCCESS; } static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) { @@ -63,38 +66,39 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { const int32_t MAX_ROWS_TO_RETURN = 4096; + int32_t vgId = TD_VID(pTq->pVnode); + int32_t code = 0; + int32_t totalRows = 0; const STqExecHandle* pExec = &pHandle->execHandle; - - qTaskInfo_t task = pExec->task; - int32_t vgId = TD_VID(pTq->pVnode); + qTaskInfo_t task = pExec->task; if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { tqDebug("prepare scan failed, return, consumer:0x%"PRIx64, pHandle->consumerId); if (pOffset->type == TMQ_OFFSET__LOG) { pRsp->rspOffset = *pOffset; - return 0; + return code; } else { tqOffsetResetToLog(pOffset, pHandle->snapshotVer); if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { tqDebug("prepare scan failed, return, consumer:0x%"PRIx64, pHandle->consumerId); pRsp->rspOffset = *pOffset; - return 0; + return code; } } } - int32_t totalRows = 0; - while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; - tqDebug("vgId:%d, tmq task start to execute, consumer:0x%"PRIx64, vgId, pHandle->consumerId); - if (qExecTask(task, &pDataBlock, &ts) < 0) { + tqDebug("vgId:%d, tmq task start to execute, consumer:0x%" PRIx64, vgId, pHandle->consumerId); + + code = qExecTask(task, &pDataBlock, &ts); + if (code != TSDB_CODE_SUCCESS) { tqError("vgId:%d, task exec error since %s, consumer:0x%" PRIx64, vgId, terrstr(), pHandle->consumerId); - return -1; + return code; } // current scan should be stopped ASAP, since the re-balance occurs. @@ -102,7 +106,12 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs break; } - tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d, failed to add block to rsp msg", vgId); + return code; + } + pRsp->blockNum++; tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%d, total blocks:%d", vgId, pHandle->consumerId, @@ -116,25 +125,25 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs } } - if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { - return -1; - } + qStreamExtractOffset(task, &pRsp->rspOffset); if (pRsp->rspOffset.type == 0) { + code = TSDB_CODE_INVALID_PARA; tqError("vgId:%d, expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, vgId, pRsp->rspOffset.type, pRsp->rspOffset.ts, pRsp->rspOffset.uid, pRsp->rspOffset.version); - return -1; + return code; } if (pRsp->withTbName || pRsp->withSchema) { + code = TSDB_CODE_INVALID_PARA; tqError("vgId:%d, get column should not with meta:%d,%d", vgId, pRsp->withTbName, pRsp->withSchema); - return -1; + return code; } tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%d, total blocks:%d, rows:%d", vgId, pHandle->consumerId, pRsp->blockNum, totalRows); - return 0; + return code; } int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { From 22ad3129b92b089b52d14c894a5b66ecf4a46476 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 15:21:47 +0800 Subject: [PATCH 030/176] fix: query schema is old issue --- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 21 +- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 229 +++++++++++++----- source/libs/executor/src/scanoperator.c | 59 ++++- source/libs/executor/src/sysscanoperator.c | 2 +- source/libs/scalar/src/sclfunc.c | 6 +- tests/parallel_test/cases.task | 3 + tests/system-test/2-query/columnLenUpdated.py | 181 ++++++++++++++ 9 files changed, 432 insertions(+), 73 deletions(-) create mode 100644 tests/system-test/2-query/columnLenUpdated.py diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 84d696e518..99fffa2cf1 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -178,7 +178,7 @@ int32_t getJsonValueLen(const char* data); int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); -int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows); +int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3c8d394b43..6e2b650b5b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -147,9 +147,17 @@ int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { return TSDB_CODE_SUCCESS; } -static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData, - int32_t itemLen, int32_t numOfRows) { - ASSERT(pColumnInfoData->info.bytes >= itemLen); +static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData, + int32_t itemLen, int32_t numOfRows, bool trimValue) { + if (pColumnInfoData->info.bytes < itemLen) { + uWarn("column/tag actual data len %d is bigger than schema len %d, trim it:%d", itemLen, pColumnInfoData->info.bytes, trimValue); + if (trimValue) { + itemLen = pColumnInfoData->info.bytes; + } else { + return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + } + } + size_t start = 1; // the first item @@ -178,10 +186,12 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren pColumnInfoData->varmeta.length += numOfRows * itemLen; } + + return TSDB_CODE_SUCCESS; } int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, - uint32_t numOfRows) { + uint32_t numOfRows, bool trimValue) { int32_t len = pColumnInfoData->info.bytes; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { len = varDataTLen(pData); @@ -193,8 +203,7 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, } } - doCopyNItems(pColumnInfoData, currentRow, pData, len, numOfRows); - return TSDB_CODE_SUCCESS; + return doCopyNItems(pColumnInfoData, currentRow, pData, len, numOfRows, trimValue); } static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource, diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2d053d04ae..7864d80a38 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -182,7 +182,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL void tsdbReaderSetId(STsdbReader* pReader, const char* idstr); void tsdbReaderClose(STsdbReader *pReader); -bool tsdbNextDataBlock(STsdbReader *pReader); +int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 96bce02b67..e11a80efb3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -949,14 +949,17 @@ static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int pDumpInfo->lastKey = maxKey + step; } -static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, +static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, SBlockLoadSuppInfo* pSup) { if (IS_VAR_DATA_TYPE(pColVal->type)) { if (!COL_VAL_IS_VALUE(pColVal)) { colDataSetNULL(pColInfoData, rowIndex); } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); - ASSERT(pColVal->value.nData <= pColInfoData->info.bytes); + if (pColVal->value.nData > pColInfoData->info.bytes) { + tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData, pColInfoData->info.bytes); + return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + } if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); } @@ -966,6 +969,8 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ } else { colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value, !COL_VAL_IS_VALUE(pColVal)); } + + return TSDB_CODE_SUCCESS; } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { @@ -1167,6 +1172,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { SDataBlk* pBlock = getCurrentBlock(pBlockIter); SSDataBlock* pResBlock = pReader->pResBlock; int32_t numOfOutputCols = pSupInfo->numOfCols; + int32_t code = TSDB_CODE_SUCCESS; SColVal cv = {0}; int64_t st = taosGetTimestampUs(); @@ -1244,7 +1250,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { } else { // varchar/nchar type for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) { tColDataGetValue(pData, j, &cv); - doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + if (code) { + return code; + } } } } @@ -1776,23 +1785,29 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* } static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key, - SFileBlockDumpInfo* pDumpInfo) { + SFileBlockDumpInfo* pDumpInfo, bool *copied) { // opt version // 1. it is not a border point // 2. the direct next point is not an duplicated timestamp + int32_t code = TSDB_CODE_SUCCESS; + + *copied = false; bool asc = (pReader->order == TSDB_ORDER_ASC); if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) { int32_t step = pReader->order == TSDB_ORDER_ASC ? 1 : -1; int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; if (nextKey != key) { // merge is not needed - doAppendRowFromFileBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + if (code) { + return code; + } pDumpInfo->rowIndex += step; - return true; + *copied = true; } } - return false; + return code; } static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, @@ -1819,20 +1834,34 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, - STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader) { + STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, bool *copied) { + int32_t code = TSDB_CODE_SUCCESS; + + *copied = false; + bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo, &pReader->verRange); if (hasVal) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 != ts) { - doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); - return true; + code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + if (code) { + return code; + } + + *copied = true; + return code; } } else { - doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); - return true; + code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + if (code) { + return code; + } + + *copied = true; + return code; } - return false; + return code; } static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid) { @@ -2022,11 +2051,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; + + return code; } static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, @@ -2034,7 +2064,8 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, bool mergeBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - + bool copied = false; + int32_t code = TSDB_CODE_SUCCESS; SRow* pTSRow = NULL; SRowMerger merge = {0}; TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); @@ -2042,7 +2073,12 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) { + code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + if (code) { + return code; + } + + if (copied) { pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { @@ -2060,10 +2096,15 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } } else { // not merge block data int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); @@ -2083,10 +2124,14 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return TSDB_CODE_SUCCESS; @@ -2131,7 +2176,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); @@ -2353,7 +2398,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); @@ -2508,7 +2553,13 @@ bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + bool copied = false; + int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied); + if (code) { + return code; + } + + if (copied) { pBlockScanInfo->lastKey = key; return TSDB_CODE_SUCCESS; } else { @@ -2528,11 +2579,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; + return code; } } @@ -2649,7 +2700,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && pBlock->nRow <= pReader->capacity) { if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { - copyBlockDataToSDataBlock(pReader); + code = copyBlockDataToSDataBlock(pReader); + if (code) { + goto _end; + } // record the last key value pBlockScanInfo->lastKey = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; @@ -2696,8 +2750,11 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { break; } - buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); - + code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + if (code) { + goto _end; + } + // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); @@ -2922,6 +2979,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; STableUidList* pUidList = &pStatus->uidList; + int32_t code = TSDB_CODE_SUCCESS; if (taosHashGetSize(pStatus->pTableMap) == 0) { return TSDB_CODE_SUCCESS; @@ -2952,7 +3010,11 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { break; } - buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (code) { + return code; + } + if (pResBlock->info.rows >= pReader->capacity) { break; } @@ -3032,7 +3094,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { break; } - buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (code) { + return code; + } + if (pResBlock->info.rows >= pReader->capacity) { break; } @@ -3784,6 +3850,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; int64_t uid = pScanInfo->uid; + int32_t code = TSDB_CODE_SUCCESS; int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); @@ -3806,7 +3873,10 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); tRowGet(pTSRow, pSchema, j, &colVal); - doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); + code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); + if (code) { + return code; + } i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { @@ -3836,6 +3906,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t rowIndex) { int32_t i = 0, j = 0; int32_t outputRowIndex = pResBlock->info.rows; + int32_t code = TSDB_CODE_SUCCESS; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; if (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -3858,7 +3929,10 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]); if (pData->cid == pSupInfo->colId[i]) { tColDataGetValue(pData, rowIndex, &cv); - doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); + code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); + if (code) { + return code; + } j += 1; } else if (pData->cid > pCol->info.colId) { // the specified column does not exist in file block, fill with null data @@ -3882,6 +3956,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader) { SSDataBlock* pBlock = pReader->pResBlock; + int32_t code = TSDB_CODE_SUCCESS; do { // SRow* pTSRow = NULL; @@ -3893,13 +3968,20 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e } if (row.type == TSDBROW_ROW_FMT) { - doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); if (freeTSRow) { taosMemoryFree(row.pTSRow); } + + if (code) { + return code; + } } else { - doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); + code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); + if (code) { + break; + } } // no data in buffer, return immediately @@ -3912,7 +3994,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e } } while (1); - return TSDB_CODE_SUCCESS; + return code; } // TODO refactor: with createDataBlockScanInfo @@ -4394,43 +4476,52 @@ _err: return code; } -static bool doTsdbNextDataBlock(STsdbReader* pReader) { +static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { + int32_t code = TSDB_CODE_SUCCESS; + // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; blockDataCleanup(pBlock); + *hasNext = false; + SReaderStatus* pStatus = &pReader->status; if (taosHashGetSize(pStatus->pTableMap) == 0) { - return false; + return code; } if (pStatus->loadFromFile) { - int32_t code = buildBlockFromFiles(pReader); + code = buildBlockFromFiles(pReader); if (code != TSDB_CODE_SUCCESS) { - return false; + return code; } if (pBlock->info.rows > 0) { - return true; + *hasNext = true; } else { resetTableListIndex(&pReader->status); - buildBlockFromBufferSequentially(pReader); - return pBlock->info.rows > 0; + code = buildBlockFromBufferSequentially(pReader); } } else { // no data in files, let's try the buffer - buildBlockFromBufferSequentially(pReader); - return pBlock->info.rows > 0; + code = buildBlockFromBufferSequentially(pReader); + *hasNext = pBlock->info.rows > 0; } + + return code; } -bool tsdbNextDataBlock(STsdbReader* pReader) { +int32_t tsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { + int32_t code = TSDB_CODE_SUCCESS; + + *hasNext = false; + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT) { - return false; + return code; } SReaderStatus* pStatus = &pReader->status; - int32_t code = tsdbAcquireReader(pReader); + code = tsdbAcquireReader(pReader); qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); if (pReader->suspended) { @@ -4438,16 +4529,21 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { } if (pReader->innerReader[0] != NULL && pReader->step == 0) { - bool ret = doTsdbNextDataBlock(pReader->innerReader[0]); + code = doTsdbNextDataBlock(pReader->innerReader[0], hasNext); + if (code) { + tsdbReleaseReader(pReader); + return code; + } + pReader->step = EXTERNAL_ROWS_PREV; - if (ret) { + if (*hasNext) { pStatus = &pReader->innerReader[0]->status; if (pStatus->composedDataBlock) { qTrace("tsdb/read: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); } - return ret; + return code; } } @@ -4464,14 +4560,19 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { pReader->step = EXTERNAL_ROWS_MAIN; } - bool ret = doTsdbNextDataBlock(pReader); - if (ret) { + code = doTsdbNextDataBlock(pReader, hasNext); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + + if (*hasNext) { if (pStatus->composedDataBlock) { qTrace("tsdb/read: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); } - return ret; + return code; } if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) { @@ -4483,23 +4584,28 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { return code; } - ret = doTsdbNextDataBlock(pReader->innerReader[1]); + code = doTsdbNextDataBlock(pReader->innerReader[1], hasNext); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + pReader->step = EXTERNAL_ROWS_NEXT; - if (ret) { + if (*hasNext) { pStatus = &pReader->innerReader[1]->status; if (pStatus->composedDataBlock) { qTrace("tsdb/read: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); } - return ret; + return code; } } qTrace("tsdb/read: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); - return false; + return code; } static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { @@ -4644,20 +4750,27 @@ STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, co static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); if (pBlockScanInfo == NULL) { return NULL; } - int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); + code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); + if (code != TSDB_CODE_SUCCESS) { + tBlockDataDestroy(&pStatus->fileBlockData); + terrno = code; + return NULL; + } + + code = copyBlockDataToSDataBlock(pReader); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData); terrno = code; return NULL; } - copyBlockDataToSDataBlock(pReader); return pReader->pResBlock; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5dff1abb97..870c96811b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -584,10 +584,16 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int if (isNullVal) { colDataSetNNULL(pColInfoData, 0, pBlock->info.rows); } else if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) { - colDataSetNItems(pColInfoData, 0, data, pBlock->info.rows); + code = colDataSetNItems(pColInfoData, 0, data, pBlock->info.rows, false); if (IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) { taosMemoryFree(data); } + if (code) { + if (freeReader) { + metaReaderClear(&mr); + } + return code; + } } else { // todo opt for json tag for (int32_t i = 0; i < pBlock->info.rows; ++i) { colDataSetVal(pColInfoData, i, data, false); @@ -634,10 +640,22 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSDataBlock* pBlock = pTableScanInfo->pResBlock; + bool hasNext = false; + int32_t code = TSDB_CODE_SUCCESS; int64_t st = taosGetTimestampUs(); - while (tsdbNextDataBlock(pTableScanInfo->base.dataReader)) { + while (true) { + code = tsdbNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + if (code) { + tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + T_LONG_JMP(pTaskInfo->env, code); + } + + if (!hasNext) { + break; + } + if (isTaskKilled(pTaskInfo)) { tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); @@ -1015,7 +1033,15 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU return NULL; } - if (tsdbNextDataBlock(pReader)) { + bool hasNext = false; + code = tsdbNextDataBlock(pReader, &hasNext); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + T_LONG_JMP(pTaskInfo->env, code); + return NULL; + } + + if (hasNext) { /*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); @@ -2104,12 +2130,22 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamRawScanInfo* pInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta pTaskInfo->streamInfo.metaRsp.metaRsp = NULL; qDebug("tmqsnap doRawScan called"); if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { - if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) { + bool hasNext = false; + if (pInfo->dataReader) { + code = tsdbNextDataBlock(pInfo->dataReader, &hasNext); + if (code) { + tsdbReleaseDataBlock(pInfo->dataReader); + longjmp(pTaskInfo->env, code); + } + } + + if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { tsdbReleaseDataBlock(pInfo->dataReader); longjmp(pTaskInfo->env, pTaskInfo->code); @@ -2610,8 +2646,21 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pInfo->base.dataReader = source->dataReader; STsdbReader* reader = pInfo->base.dataReader; + bool hasNext = false; qTrace("tsdb/read-table-data: %p, enter next reader", reader); - while (tsdbNextDataBlock(reader)) { + + while (true) { + code = tsdbNextDataBlock(reader, &hasNext); + if (code != 0) { + tsdbReleaseDataBlock(reader); + pInfo->base.dataReader = NULL; + T_LONG_JMP(pTaskInfo->env, code); + } + + if (!hasNext) { + break; + } + if (isTaskKilled(pTaskInfo)) { tsdbReleaseDataBlock(reader); pInfo->base.dataReader = NULL; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index f24d3523c8..00a90cc108 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1627,7 +1627,7 @@ static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScan char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(varTbName, name); - colDataSetNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows); + colDataSetNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows, true); } doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 195a08525c..1affbac613 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1755,7 +1755,11 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { char* p = colDataGetVarData(pInput->columnData, 0); - colDataSetNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows); + int32_t code = colDataSetNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows, true); + if (code) { + return code; + } + pOutput->numOfRows += pInput->numOfRows; return TSDB_CODE_SUCCESS; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 58fa0cbfab..6df0651f96 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -14,6 +14,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 @@ -22,6 +24,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py new file mode 100644 index 0000000000..d6940fde8b --- /dev/null +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -0,0 +1,181 @@ + +import taos +import sys +import time +import socket +import os +import platform +if platform.system().lower() == 'windows': + import wexpect as taosExpect +else: + import pexpect as taosExpect + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +def taos_command (buildPath, key, value, expectString, sqlString=''): + if len(key) == 0: + tdLog.exit("taos test key is null!") + + if platform.system().lower() == 'windows': + taosCmd = buildPath + '\\build\\bin\\taos.exe ' + taosCmd = taosCmd.replace('\\','\\\\') + else: + taosCmd = buildPath + '/build/bin/taos ' + + cfgPath = buildPath + "/../sim/psim/cfg" + taosCmd = taosCmd + ' -c' + cfgPath + ' -' + key + if len(value) != 0: + taosCmd = taosCmd + ' ' + value + + tdLog.info ("taos cmd: %s" % taosCmd) + + child = taosExpect.spawn(taosCmd, timeout=20) + #output = child.readline() + #print (output.decode()) + if len(expectString) != 0: + i = child.expect([expectString, taosExpect.TIMEOUT, taosExpect.EOF], timeout=20) + else: + i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=20) + + if platform.system().lower() == 'windows': + retResult = child.before + else: + retResult = child.before.decode() + print(retResult) + #print(child.after.decode()) + if i == 0: + print ('taos login success! Here can run sql, taos> ') + return "TAOS_OK" + else: + return "TAOS_FAIL" + +class TDTestCase: + #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ + # 'serverPort': 7080, 'firstEp': 'trd02:7080'} + hostname = socket.gethostname() + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP + serverPort = '7080' + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["serverPort"] = serverPort + clientCfgDict["firstEp"] = hostname + ':' + serverPort + clientCfgDict["secondEp"] = hostname + ':' + serverPort + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + clientCfgDict["fqdn"] = hostname + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["clientCfg"] = clientCfgDict + updatecfgDict["serverPort"] = serverPort + updatecfgDict["firstEp"] = hostname + ':' + serverPort + updatecfgDict["secondEp"] = hostname + ':' + serverPort + updatecfgDict["fqdn"] = hostname + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + tdSql.query("create user testpy pass 'testpy'") + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] + netrole = ['client', 'server'] + + keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} + + keyDict['h'] = self.hostname + keyDict['c'] = cfgPath + keyDict['P'] = self.serverPort + + tdSql.query("drop database if exists db1") + tdSql.query("create database if not exists db1 vgroups 1") + tdSql.query("use db1") + tdSql.query("create table tba (ts timestamp, f1 binary(2))") + tdSql.query("insert into tba values (now, '22')") + tdSql.query("select * from tba") + tdSql.checkData(0, 1, '22') + + keyDict['s'] = "\"alter table db1.tba modify column f1 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"insert into db1.tba values (now, '55555')\"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select * from tba order by ts") + tdSql.checkData(0, 1, '22') + tdSql.checkData(1, 1, '55555') + + + tdSql.query("create table stb (ts timestamp, f1 int) tags (tg1 binary(2))") + tdSql.query("create table tb1 using stb tags('bb')") + tdSql.query("insert into tb1 values (now, 2)") + tdSql.query("select count(*) from stb group by tg1") + tdSql.checkData(0, 0, 1) + + keyDict['s'] = "\"alter table db1.stb modify tag tg1 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"create table db1.tb2 using db1.stb tags('bbbbb')\"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"insert into db1.tb2 values (now, 2)\"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select count(*) from stb group by tg1") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 38b5f4ae1b545c5f55dd3640c82421d0881332e5 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Tue, 28 Mar 2023 15:34:48 +0800 Subject: [PATCH 031/176] fix: taosToolsInstallDir if condition order (#20665) --- packaging/tools/makepkg.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 29160238ce..a2dec155e8 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -51,9 +51,9 @@ fi if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then cd ${top_dir}/tools/taos-tools/packaging/deb - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - + taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1) + [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}" cd ${curr_dir} From 7342aeb750c09d6c89915ea766457dd45468de96 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 15:42:53 +0800 Subject: [PATCH 032/176] fix(tmq): fix the syntax error. --- source/dnode/vnode/src/tq/tqScan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index c01108f114..1a166d326f 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -140,7 +140,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs return code; } - tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%d, total blocks:%d, rows:%d", vgId, pHandle->consumerId, + tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, total blocks:%d, rows:%d", vgId, pHandle->consumerId, pRsp->blockNum, totalRows); return code; From 0ae2034ebbc96472ee53df2ed90341c9bd7baf1b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 16:05:05 +0800 Subject: [PATCH 033/176] fix: fix syntax error. --- source/dnode/vnode/src/tq/tqPush.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 73d722dba5..1619829115 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -300,7 +300,7 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to copy data for stream since out of memory, vgId:%d, consumer:0x%"PRIx64, vgId); + tqError("failed to copy data for stream since out of memory, vgId:%d", vgId); taosWUnLockLatch(&pTq->lock); return -1; } From b5a1246fea538de8b87e89f7de73bc5e98a2be2f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 28 Mar 2023 16:52:54 +0800 Subject: [PATCH 034/176] fix: post script add jemalloc (#20672) * fix: tools/post.sh to ldconfig for jemalloc * fix: rpm spec for jemalloc --- packaging/rpm/tdengine.spec | 38 +++++++++----------------- packaging/tools/post.sh | 54 +++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 25 deletions(-) diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index e9d86219cb..c21063e6a4 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -90,45 +90,33 @@ cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepat cp -r %{_compiledir}/../examples/* %{buildroot}%{homepath}/examples if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then - mkdir -p %{buildroot}%{userlocalpath}/bin - mkdir -p %{buildroot}%{userlocalpath}/lib - mkdir -p %{buildroot}%{userlocalpath}/lib/pkgconfig - mkdir -p %{buildroot}%{userlocalpath}/include - mkdir -p %{buildroot}%{userlocalpath}/include/jemalloc - mkdir -p %{buildroot}%{userlocalpath}/share - mkdir -p %{buildroot}%{userlocalpath}/share/doc - mkdir -p %{buildroot}%{userlocalpath}/share/doc/jemalloc - mkdir -p %{buildroot}%{userlocalpath}/share/man - mkdir -p %{buildroot}%{userlocalpath}/share/man/man3 + mkdir -p %{buildroot}%{homepath}/jemalloc/ ||: + mkdir -p %{buildroot}%{homepath}/jemalloc/include/jemalloc/ ||: + mkdir -p %{buildroot}%{homepath}/jemalloc/lib/ ||: + mkdir -p %{buildroot}%{homepath}/jemalloc/lib/pkgconfig ||: - cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{userlocalpath}/bin/ + cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{homepath}/jemalloc/bin if [ -f %{_compiledir}/build/bin/jemalloc.sh ]; then - cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{userlocalpath}/bin/ + cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{homepath}/jemalloc/bin fi if [ -f %{_compiledir}/build/bin/jeprof ]; then - cp %{_compiledir}/build/bin/jeprof %{buildroot}%{userlocalpath}/bin/ + cp %{_compiledir}/build/bin/jeprof %{buildroot}%{homepath}/jemalloc/bin fi if [ -f %{_compiledir}/build/include/jemalloc/jemalloc.h ]; then - cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{userlocalpath}/include/jemalloc/ + cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{homepath}/jemalloc/include/jemalloc/ fi if [ -f %{_compiledir}/build/lib/libjemalloc.so.2 ]; then - cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/ - ln -sf libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/libjemalloc.so + cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib + ln -sf libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib/libjemalloc.so fi if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then - cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{userlocalpath}/lib/ + cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{homepath}/jemalloc/lib fi if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then - cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{userlocalpath}/lib/ + cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{homepath}/jemalloc/lib fi if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then - cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{userlocalpath}/lib/pkgconfig/ - fi - if [ -f %{_compiledir}/build/share/doc/jemalloc/jemalloc.html ]; then - cp %{_compiledir}/build/share/doc/jemalloc/jemalloc.html %{buildroot}%{userlocalpath}/share/doc/jemalloc/ - fi - if [ -f %{_compiledir}/build/share/man/man3/jemalloc.3 ]; then - cp %{_compiledir}/build/share/man/man3/jemalloc.3 %{buildroot}%{userlocalpath}/share/man/man3/ + cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig fi fi diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index c9fab51b28..35c28feb6a 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -145,6 +145,59 @@ function install_include() { log_print "install include success" } +function install_jemalloc() { + jemalloc_dir=${script_dir}/../jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo}/usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}/usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi +} + function install_lib() { log_print "start install lib from ${lib_dir} to ${lib_link_dir}" ${csudo}rm -f ${lib_link_dir}/libtaos* || : @@ -663,6 +716,7 @@ function install_TDengine() { # Install include, lib, binary and service install_include && install_lib && + install_jemalloc install_bin if [[ "$?" != 0 ]];then From b11a105327f3154524bfd9cdf79d6633257dc001 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 28 Mar 2023 17:36:23 +0800 Subject: [PATCH 035/176] fix:[TD-23339] parse block error --- source/libs/parser/src/parInsertUtil.c | 53 ++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 0ed32afbbc..90a40ba885 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -631,10 +631,10 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate ret = TSDB_CODE_INVALID_PARA; goto end; } - for (int c = 0; c < boundInfo->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[c]; - SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); - if(tFields == NULL || findFileds(pColSchema, tFields, numFields)){ + if(tFields == NULL){ + for (int j = 0; j < boundInfo->numOfBound; j++){ + SSchema* pColSchema = &pSchema[j]; + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { uError("type or bytes not equal"); ret = TSDB_CODE_INVALID_PARA; @@ -652,12 +652,49 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); fields += sizeof(int8_t) + sizeof(int32_t); if (needChangeLength) { - pStart += htonl(colLength[c]); + pStart += htonl(colLength[j]); } else { - pStart += colLength[c]; + pStart += colLength[j]; + } + } + }else{ + for (int i = 0; i < numFields; i++) { + for (int j = 0; j < boundInfo->numOfBound; j++){ + SSchema* pColSchema = &pSchema[j]; + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); + if(strcmp(pSchema->name, tFields[i].name) == 0){ + if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + uError("type or bytes not equal"); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } + + int8_t* offset = pStart; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + pStart += numOfRows * sizeof(int32_t); + } else { + pStart += BitmapLen(numOfRows); + } + char* pData = pStart; + + tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); + fields += sizeof(int8_t) + sizeof(int32_t); + if (needChangeLength) { + pStart += htonl(colLength[i]); + } else { + pStart += colLength[i]; + } + boundInfo->pColIndex[j] = -1; + } + } + + } + + for (int c = 0; c < boundInfo->numOfBound; ++c) { + if( boundInfo->pColIndex[c] != -1){ + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); + tColDataAddValueByDataBlock(pCol, 0, 0, numOfRows, NULL, NULL); } - }else{ - tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, NULL, NULL); } } From a87d35be082d1409a1c9a5d384f59472b1b3cec9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 17:40:46 +0800 Subject: [PATCH 036/176] fix: optimize performance --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/inc/tsdb.h | 2 ++ source/dnode/vnode/src/tsdb/tsdbMemTable.c | 30 ++++++++++++++++++- source/dnode/vnode/src/tsdb/tsdbRead.c | 34 +++++++++++++++++++--- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executor.c | 4 +-- source/libs/executor/src/scanoperator.c | 13 +++++++-- source/libs/executor/src/sysscanoperator.c | 2 +- 8 files changed, 76 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2d053d04ae..eae398880b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -178,7 +178,7 @@ typedef struct STsdbReader STsdbReader; int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly); void tsdbReaderSetId(STsdbReader* pReader, const char* idstr); void tsdbReaderClose(STsdbReader *pReader); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 452b1f6c0b..0c4ada2cb1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -224,6 +224,8 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, void *tsdbTbDataIterDestroy(STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter); +void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum); + // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); // tsdbFile.c ============================================================================================== diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d34af9acae..46d0509c38 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -282,6 +282,34 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return true; } +int64_t tsdbCountTbDataRows(STbData *pTbData) { + SMemSkipListNode *pNode = NULL; + int64_t rowsNum = 0; + + while (true) { + pNode = SL_GET_NODE_FORWARD(pTbData->sl.pHead, 0); + if (pNode == pTbData->sl.pTail) { + return rowsNum; + } + + rowsNum++; + } +} + +void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) { + taosRLockLatch(&pMemTable->latch); + for (int32_t i = 0; i < pMemTable->nBucket; ++i) { + STbData *pTbData = pMemTable->aBucket[i]; + + void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); + if (p == NULL) { + continue; + } + rowsNum += tsdbCountTbDataRows(pTbData); + } + taosRUnLockLatch(&pMemTable->latch); +} + static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { int32_t code = 0; @@ -787,4 +815,4 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { _exit: return aTbDataP; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c6444b5ded..8b6f318434 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3113,7 +3113,7 @@ static int32_t doSumBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { } STableBlockScanInfo** p = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); - if (p == NULL || *p == NULL) { + if (p == NULL) { continue; } @@ -3189,6 +3189,10 @@ static int32_t readRowsCountFromStt(STsdbReader* pReader) { taosArrayClear(pBlockLoadInfo->aSttBlk); continue; } + for (int32_t i = 0; i < size; ++i) { + SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + pReader->rowsNum += p->nRow; + } } else { for (int32_t i = 0; i < size; ++i) { SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); @@ -3210,6 +3214,22 @@ static int32_t readRowsCountFromStt(STsdbReader* pReader) { return code; } +static int32_t readRowsCountFromMem(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t memNum = 0, imemNum = 0; + if (pReader->pReadSnap->pMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pMem, pReader->status.pTableMap, &memNum); + } + + if (pReader->pReadSnap->pIMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pIMem, pReader->status.pTableMap, &imemNum); + } + + pReader->rowsNum += memNum + imemNum; + + return code; +} + static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; @@ -4170,7 +4190,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) { + SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr, bool countOnly) { STimeWindow window = pCond->twindows; if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { pCond->twindows.skey += 1; @@ -4262,9 +4282,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pReader->suspended = true; + if (countOnly) { + pReader->readMode = READ_MODE_COUNT_ONLY; + } - pReader->readMode = READ_MODE_COUNT_ONLY; - tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -4588,6 +4609,11 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { } } + code = readRowsCountFromMem(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + pBlock->info.rows = pReader->rowsNum; pBlock->info.id.uid = 0; pBlock->info.dataLoad = 0; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 3f519568c4..fd29df1acc 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -339,6 +339,7 @@ typedef struct STableScanInfo { int8_t scanMode; int8_t assignBlockUid; bool hasGroupByTag; + bool countOnly; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index db58bd6f68..a1b27114b3 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1140,7 +1140,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num, - pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 || + pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL, false) < 0 || pTableScanInfo->base.dataReader == NULL) { qError("tsdbReaderOpen failed. uid:%" PRIi64, pOffset->uid); return -1; @@ -1192,7 +1192,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL); + tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL, false); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2af13f083d..854a2c1aaf 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -31,6 +31,9 @@ #include "thash.h" #include "ttypes.h" +int32_t scanDebug = 0; + + #define MULTI_READER_MAX_TABLE_NUM 5000 #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) @@ -785,7 +788,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { ASSERT(pInfo->base.dataReader == NULL); int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, - (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo)); + (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -916,6 +919,10 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } + if (scanDebug) { + pInfo->countOnly = true; + } + taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, optrDefaultBufFn, getTableScannerExecInfo); @@ -1007,7 +1014,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, - (STsdbReader**)&pReader, GET_TASKID(pTaskInfo)); + (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -2601,7 +2608,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { - code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo)); + code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index f24d3523c8..d05c692f19 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2267,7 +2267,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str); + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; From 78ab9d3bf9a8f38aecec1eb3eb20c61139e2fe58 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 28 Mar 2023 17:11:40 +0800 Subject: [PATCH 037/176] fix: process appendLog replies in vnode-sync thread --- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 4 ++-- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index ce485cb6ca..c2c9e37c8c 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -192,16 +192,16 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 84dfab7e6c..28cb5d2058 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -553,17 +553,17 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; From 5c9638d2c431d1e118bcb3b1c1443b605f557ef7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 28 Mar 2023 18:09:19 +0800 Subject: [PATCH 038/176] fix:modify test case for tmq --- tests/system-test/7-tmq/subscribeStb.py | 4 ++-- tests/system-test/7-tmq/subscribeStb1.py | 4 ++-- tests/system-test/7-tmq/subscribeStb2.py | 9 ++++++--- tests/system-test/7-tmq/subscribeStb3.py | 18 +++++++++--------- tests/system-test/7-tmq/subscribeStb4.py | 4 ++-- 5 files changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeStb.py b/tests/system-test/7-tmq/subscribeStb.py index 3ff0b25ff6..b9a6e7667d 100644 --- a/tests/system-test/7-tmq/subscribeStb.py +++ b/tests/system-test/7-tmq/subscribeStb.py @@ -226,7 +226,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -307,7 +307,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py index 3dc3528d04..63ba2c8c16 100644 --- a/tests/system-test/7-tmq/subscribeStb1.py +++ b/tests/system-test/7-tmq/subscribeStb1.py @@ -233,7 +233,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -320,7 +320,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index ae771d0556..dbfbc6f7bc 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -233,7 +233,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 5 + pollDelay = 10 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -251,6 +251,7 @@ class TDTestCase: tdLog.info("start consume 1 processor") self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + tdLog.sleep(2) tdLog.info("start one new thread to insert data") parameterDict['actionType'] = actionType.INSERT_DATA @@ -271,6 +272,7 @@ class TDTestCase: tdLog.info("start consume 2 processor") self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + tdLog.sleep(2) tdLog.info("start one new thread to insert data") parameterDict['actionType'] = actionType.INSERT_DATA @@ -338,7 +340,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 100 + pollDelay = 20 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -380,6 +382,7 @@ class TDTestCase: tdLog.info("start consume 2 processor") self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + tdLog.sleep(2) tdLog.info("start one new thread to insert data") parameterDict['actionType'] = actionType.INSERT_DATA @@ -394,7 +397,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt*2: + if totalConsumeRows < expectrowcnt*2: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 025f403282..33a7c506c5 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -233,7 +233,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -269,7 +269,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt-10000: + if totalConsumeRows < expectrowcnt-10000: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt-10000)) tdLog.exit("tmq consume rows error!") @@ -346,7 +346,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -433,7 +433,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -445,7 +445,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -467,7 +467,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -520,7 +520,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -532,7 +532,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -555,7 +555,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt*(1/2+1/4): + if totalConsumeRows < expectrowcnt*(1/2+1/4): tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*(1/2+1/4))) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb4.py b/tests/system-test/7-tmq/subscribeStb4.py index 6aa3da66a4..c0feb8fdba 100644 --- a/tests/system-test/7-tmq/subscribeStb4.py +++ b/tests/system-test/7-tmq/subscribeStb4.py @@ -231,7 +231,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -305,7 +305,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) From 0cbcf624b4043771bcb01f44bfc1209013ce6880 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 18:24:26 +0800 Subject: [PATCH 039/176] fix: add debug --- source/common/src/tglobal.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index aeeec1d61c..e97523df3d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1347,9 +1347,15 @@ void taosCleanupCfg() { } } +extern int32_t scanDebug; void taosCfgDynamicOptions(const char *option, const char *value) { if (strncasecmp(option, "debugFlag", 9) == 0) { int32_t flag = atoi(value); + if (1 == flag) { + scanDebug = 1; + } else { + scanDebug = 0; + } taosSetAllDebugFlag(flag, true); return; } From e0f951e668f3c7cced572627124ff0491a520c1f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 19:15:38 +0800 Subject: [PATCH 040/176] fix: memory rows issue --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 46d0509c38..23c5ddcf4c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -300,12 +300,16 @@ void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int taosRLockLatch(&pMemTable->latch); for (int32_t i = 0; i < pMemTable->nBucket; ++i) { STbData *pTbData = pMemTable->aBucket[i]; - - void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); - if (p == NULL) { - continue; + while (pTbData) { + void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); + if (p == NULL) { + pTbData = pTbData->next; + continue; + } + + rowsNum += tsdbCountTbDataRows(pTbData); + pTbData = pTbData->next; } - rowsNum += tsdbCountTbDataRows(pTbData); } taosRUnLockLatch(&pMemTable->latch); } From c5f37d284ecdd2c9deab38b321f6a5a9b1fca1e9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 19:27:49 +0800 Subject: [PATCH 041/176] fix: memory row count issue --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 23c5ddcf4c..46f5aefba0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -283,11 +283,11 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { } int64_t tsdbCountTbDataRows(STbData *pTbData) { - SMemSkipListNode *pNode = NULL; + SMemSkipListNode *pNode = pTbData->sl.pHead; int64_t rowsNum = 0; while (true) { - pNode = SL_GET_NODE_FORWARD(pTbData->sl.pHead, 0); + pNode = SL_GET_NODE_FORWARD(pNode, 0); if (pNode == pTbData->sl.pTail) { return rowsNum; } From a0c8dd52b1429dc6308fc329212d73f3e7dc3b09 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 19:42:09 +0800 Subject: [PATCH 042/176] fix: memory count issue --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 46f5aefba0..bf47f25e15 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -307,7 +307,7 @@ void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int continue; } - rowsNum += tsdbCountTbDataRows(pTbData); + *rowsNum += tsdbCountTbDataRows(pTbData); pTbData = pTbData->next; } } From f1ff5dcec484f87bd07f03627731ebd492d0a3c4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 22:53:57 +0800 Subject: [PATCH 043/176] fix(query): use hashmap to keep the multiple schema. --- include/util/tsimplehash.h | 7 +++ source/dnode/vnode/src/tsdb/tsdbRead.c | 59 ++++++++++++++------------ source/util/src/tsimplehash.c | 38 ++++++++++------- 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/include/util/tsimplehash.h b/include/util/tsimplehash.h index c9df911476..987a9fe2a8 100644 --- a/include/util/tsimplehash.h +++ b/include/util/tsimplehash.h @@ -48,6 +48,13 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn); */ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj); +/** + * set the free function pointer + * @param pHashObj + * @param freeFp + */ +void tSimpleHashSetFreeFp(SSHashObj* pHashObj, _hash_free_fn_t freeFp); + int32_t tSimpleHashPrint(const SSHashObj *pHashObj); /** diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 89d3239c33..3d8b2a4031 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -15,6 +15,7 @@ #include "osDef.h" #include "tsdb.h" +#include "tsimplehash.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -177,7 +178,8 @@ struct STsdbReader { STsdbReadSnap* pReadSnap; SIOCostSummary cost; STSchema* pSchema; // the newest version schema - STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times +// STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times + SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema SDataFReader* pFileReader; // the file reader SDelFReader* pDelFReader; // the del file reader SArray* pDelIdx; // del file block index; @@ -1858,28 +1860,23 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* return pReader->pSchema; } - if (pReader->pMemSchema == NULL) { - int32_t code = - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; - } else { - return pReader->pMemSchema; - } + void** p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion)); + if (p != NULL) { + return *(STSchema**) p; } - if (pReader->pMemSchema->version == sversion) { - return pReader->pMemSchema; - } - - taosMemoryFreeClear(pReader->pMemSchema); - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); - if (code != TSDB_CODE_SUCCESS || pReader->pMemSchema == NULL) { + STSchema* ptr = NULL; + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); + if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; } else { - return pReader->pMemSchema; + code = tSimpleHashPut(pReader->pSchemaMap, &sversion, sizeof(sversion), &ptr, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + return ptr; } } @@ -4002,6 +3999,11 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { return code; } +static void freeSchemaFunc(void* param) { + void* p = *(void**) param; + taosMemoryFree(p); +} + // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) { @@ -4078,6 +4080,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } } + pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); + if (pReader->pSchemaMap == NULL) { + tsdbError("failed init schema hash for reader", pReader->idStr); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc); if (pReader->pSchema != NULL) { code = updateBlockSMAInfo(pReader->pSchema, &pReader->suppInfo); if (code != TSDB_CODE_SUCCESS) { @@ -4120,7 +4130,7 @@ void tsdbReaderClose(STsdbReader* pReader) { p->status.uidList.tableUidList = NULL; p->pReadSnap = NULL; p->pSchema = NULL; - p->pMemSchema = NULL; + p->pSchemaMap = NULL; p = pReader->innerReader[1]; @@ -4128,7 +4138,7 @@ void tsdbReaderClose(STsdbReader* pReader) { p->status.uidList.tableUidList = NULL; p->pReadSnap = NULL; p->pSchema = NULL; - p->pMemSchema = NULL; + p->pSchemaMap = NULL; tsdbReaderClose(pReader->innerReader[0]); tsdbReaderClose(pReader->innerReader[1]); @@ -4208,10 +4218,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); - if (pReader->pMemSchema != pReader->pSchema) { - taosMemoryFree(pReader->pMemSchema); - } - + tSimpleHashCleanup(pReader->pSchemaMap); taosMemoryFreeClear(pReader); } @@ -4371,14 +4378,14 @@ int32_t tsdbReaderResume(STsdbReader* pReader) { pPrevReader->status.pTableMap = pReader->status.pTableMap; pPrevReader->status.uidList = pReader->status.uidList; pPrevReader->pSchema = pReader->pSchema; - pPrevReader->pMemSchema = pReader->pMemSchema; + pPrevReader->pSchemaMap = pReader->pSchemaMap; pPrevReader->pReadSnap = pReader->pReadSnap; pNextReader->capacity = 1; pNextReader->status.pTableMap = pReader->status.pTableMap; pNextReader->status.uidList = pReader->status.uidList; pNextReader->pSchema = pReader->pSchema; - pNextReader->pMemSchema = pReader->pMemSchema; + pNextReader->pSchemaMap = pReader->pSchemaMap; pNextReader->pReadSnap = pReader->pReadSnap; code = doOpenReaderImpl(pPrevReader); diff --git a/source/util/src/tsimplehash.c b/source/util/src/tsimplehash.c index 70acffed5d..310fdd27ab 100644 --- a/source/util/src/tsimplehash.c +++ b/source/util/src/tsimplehash.c @@ -28,19 +28,23 @@ #define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define FREE_HASH_NODE(_n) \ - do { \ - taosMemoryFreeClear(_n); \ +#define FREE_HASH_NODE(_n, fp) \ + do { \ + if (fp) { \ + fp((_n)->data); \ + } \ + taosMemoryFreeClear(_n); \ } while (0); struct SSHashObj { - SHNode **hashList; - size_t capacity; // number of slots - int64_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _equal_fn_t equalFp; // equal function - SArray* pHashNodeBuf;// hash node allocation buffer, 1k size of each page by default - int32_t offset; // allocation offset in current page + SHNode **hashList; + size_t capacity; // number of slots + int64_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + _hash_free_fn_t freeFp; // free function + SArray *pHashNodeBuf; // hash node allocation buffer, 1k size of each page by default + int32_t offset; // allocation offset in current page }; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { @@ -71,7 +75,6 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) { pHashObj->capacity = taosHashCapacity((int32_t)capacity); pHashObj->equalFp = memcmp; - pHashObj->pHashNodeBuf = taosArrayInit(10, sizeof(void*)); pHashObj->offset = 0; pHashObj->size = 0; @@ -92,6 +95,10 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { return (int32_t) pHashObj->size; } +void tSimpleHashSetFreeFp(SSHashObj* pHashObj, _hash_free_fn_t freeFp) { + pHashObj->freeFp = freeFp; +} + static void* doInternalAlloc(SSHashObj* pHashObj, int32_t size) { #if 0 void** p = taosArrayGetLast(pHashObj->pHashNodeBuf); @@ -306,7 +313,8 @@ int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen) { } else { pPrev->next = pNode->next; } - FREE_HASH_NODE(pNode); + + FREE_HASH_NODE(pNode, pHashObj->freeFp); pHashObj->size -= 1; code = TSDB_CODE_SUCCESS; break; @@ -341,7 +349,7 @@ int32_t tSimpleHashIterateRemove(SSHashObj *pHashObj, const void *key, size_t ke *pIter = pPrev ? GET_SHASH_NODE_DATA(pPrev) : NULL; } - FREE_HASH_NODE(pNode); + FREE_HASH_NODE(pNode, pHashObj->freeFp); pHashObj->size -= 1; break; } @@ -370,14 +378,13 @@ void tSimpleHashClear(SSHashObj *pHashObj) { while (pNode) { pNext = pNode->next; - FREE_HASH_NODE(pNode); + FREE_HASH_NODE(pNode, pHashObj->freeFp); pNode = pNext; } pHashObj->hashList[i] = NULL; } - taosArrayClearEx(pHashObj->pHashNodeBuf, destroyItems); pHashObj->offset = 0; pHashObj->size = 0; } @@ -388,7 +395,6 @@ void tSimpleHashCleanup(SSHashObj *pHashObj) { } tSimpleHashClear(pHashObj); - taosArrayDestroy(pHashObj->pHashNodeBuf); taosMemoryFreeClear(pHashObj->hashList); taosMemoryFree(pHashObj); } From ceb160c23bcf683af37e4aa54bf89345780d9f1a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 28 Mar 2023 23:14:11 +0800 Subject: [PATCH 044/176] fix(query): fix memory leak. --- source/dnode/mnode/impl/src/mndConsumer.c | 24 ++++++++++++++++++++-- source/dnode/mnode/impl/src/mndSubscribe.c | 1 + 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 0b2b5ec35c..6acf7fb98a 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -247,6 +247,13 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebInfo; } +static void freeRebalanceItem(void* param) { + SMqRebInfo* pInfo = param; + taosArrayDestroy(pInfo->lostConsumers); + taosArrayDestroy(pInfo->newConsumers); + taosArrayDestroy(pInfo->removedConsumers); +} + static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SSdb *pSdb = pMnode->pSdb; @@ -262,8 +269,21 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); + if (pRebMsg == NULL) { + mError("failed to create the rebalance msg, size:%d, quit mq timer", sizeof(SMqDoRebalanceMsg)); + mndRebEnd(); + return TSDB_CODE_OUT_OF_MEMORY; + } + pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); - // TODO set cleanfp + if (pRebMsg->rebSubHash == NULL) { + mError("failed to create rebalance hashmap"); + rpcFreeCont(pRebMsg); + mndRebEnd(); + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosHashSetFreeFp(pRebMsg->rebSubHash, freeRebalanceItem); // iterate all consumers, find all modification while (1) { @@ -356,7 +376,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); - mDebug("mq rebalance finished, no modification"); + mDebug("mq timer finished, no need to re-balance"); mndRebEnd(); } return 0; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index b8dbec2f84..739de68e5f 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -611,6 +611,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { break; } + // todo handle the malloc failure SMqRebInputObj rebInput = {0}; SMqRebOutputObj rebOutput = {0}; rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t)); From 70d0d7a63fa97cfe2191519ac4efafb2bd09092f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Mar 2023 09:08:07 +0800 Subject: [PATCH 045/176] fix(tmq): fix the syntax error. --- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 6acf7fb98a..68670eedfb 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -270,7 +270,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); if (pRebMsg == NULL) { - mError("failed to create the rebalance msg, size:%d, quit mq timer", sizeof(SMqDoRebalanceMsg)); + mError("failed to create the rebalance msg, size:%d, quit mq timer", (int32_t) sizeof(SMqDoRebalanceMsg)); mndRebEnd(); return TSDB_CODE_OUT_OF_MEMORY; } From d5a06da96e12312c247eeaa5948b21a540d6cab9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 29 Mar 2023 09:31:00 +0800 Subject: [PATCH 046/176] fix:[TD-23339] parse block error --- source/libs/parser/src/parInsertUtil.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 90a40ba885..779ba091aa 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -661,8 +661,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate for (int i = 0; i < numFields; i++) { for (int j = 0; j < boundInfo->numOfBound; j++){ SSchema* pColSchema = &pSchema[j]; - SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); - if(strcmp(pSchema->name, tFields[i].name) == 0){ + if(strcmp(pColSchema->name, tFields[i].name) == 0){ if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { uError("type or bytes not equal"); ret = TSDB_CODE_INVALID_PARA; @@ -677,6 +676,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } char* pData = pStart; + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); fields += sizeof(int8_t) + sizeof(int32_t); if (needChangeLength) { @@ -685,6 +685,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate pStart += colLength[i]; } boundInfo->pColIndex[j] = -1; + break; } } From 6f6239a7fa272e80630cf825db7deb9c2afd42b8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 29 Mar 2023 09:44:26 +0800 Subject: [PATCH 047/176] fix:[TD-23339] parse block error --- source/libs/parser/src/parInsertUtil.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 779ba091aa..132a3b2618 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -695,6 +695,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate if( boundInfo->pColIndex[c] != -1){ SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); tColDataAddValueByDataBlock(pCol, 0, 0, numOfRows, NULL, NULL); + }else{ + boundInfo->pColIndex[c] = c; // restore for next block } } } From 702202d4afa78ae892344c6b18f0c43e09435c6a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 10:14:53 +0800 Subject: [PATCH 048/176] fix: fix block scan issue --- source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e11a80efb3..5867580d2d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4496,17 +4496,16 @@ static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { return code; } - if (pBlock->info.rows > 0) { - *hasNext = true; - } else { + if (pBlock->info.rows <= 0) { resetTableListIndex(&pReader->status); code = buildBlockFromBufferSequentially(pReader); } } else { // no data in files, let's try the buffer code = buildBlockFromBufferSequentially(pReader); - *hasNext = pBlock->info.rows > 0; } + *hasNext = pBlock->info.rows > 0; + return code; } From 7a050d64bdc6573201d26af6b3046495358f0e5e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Mar 2023 10:27:05 +0800 Subject: [PATCH 049/176] fix(query): set ptr to be NULL. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 +++-- source/util/src/tsimplehash.c | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 3d8b2a4031..d9f6b66d6a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3673,7 +3673,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p TSDBKEY ik = TSDBROW_KEY(piRow); if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); int32_t code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { @@ -3686,7 +3686,8 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMerge(&merge, pRow); + pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + tsdbRowMergerAdd(&merge, pRow, pSchema); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/util/src/tsimplehash.c b/source/util/src/tsimplehash.c index 310fdd27ab..ec1991923f 100644 --- a/source/util/src/tsimplehash.c +++ b/source/util/src/tsimplehash.c @@ -75,6 +75,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) { pHashObj->capacity = taosHashCapacity((int32_t)capacity); pHashObj->equalFp = memcmp; + pHashObj->freeFp = NULL; pHashObj->offset = 0; pHashObj->size = 0; From 94c6af39daf3dfac36be05b0b62042dcfb78e3ab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Mar 2023 10:46:44 +0800 Subject: [PATCH 050/176] fix(tmq): fix the invalid free --- source/dnode/mnode/impl/src/mndSubscribe.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 739de68e5f..64a3170d47 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -596,13 +596,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqDoRebalanceMsg *pReq = pMsg->pCont; void *pIter = NULL; - bool rebalanceExec = false; // to ensure only once. + bool rebalanceOnce = false; // to ensure only once. mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash)); // here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction. while (1) { - if (rebalanceExec) { + if (rebalanceOnce) { break; } @@ -673,10 +673,6 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { mError("mq re-balance persist output error, possibly vnode splitted or dropped"); } - taosArrayDestroy(pRebInfo->lostConsumers); - taosArrayDestroy(pRebInfo->newConsumers); - taosArrayDestroy(pRebInfo->removedConsumers); - taosArrayDestroy(rebOutput.newConsumers); taosArrayDestroy(rebOutput.touchedConsumers); taosArrayDestroy(rebOutput.removedConsumers); @@ -684,7 +680,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { tDeleteSubscribeObj(rebOutput.pSub); taosMemoryFree(rebOutput.pSub); - rebalanceExec = true; + rebalanceOnce = true; } // reset flag From 0327a8eaa3450c8d33da983c081a68d9e16ce88d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 10:46:56 +0800 Subject: [PATCH 051/176] fix: block rows number type --- include/common/tcommon.h | 2 +- source/common/src/tdatablock.c | 10 +++---- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 6 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 32 ++++++++++------------ source/libs/executor/src/executorimpl.c | 4 +-- source/libs/executor/src/projectoperator.c | 4 +-- source/libs/executor/src/scanoperator.c | 24 ++++++++-------- source/libs/executor/src/sortoperator.c | 2 +- source/libs/qworker/src/qworker.c | 2 +- 11 files changed, 45 insertions(+), 45 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 9e928a79ac..51a714c792 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -185,7 +185,7 @@ typedef struct SBlockID { typedef struct SDataBlockInfo { STimeWindow window; int32_t rowSize; - int32_t rows; // todo hide this attribute + int64_t rows; // todo hide this attribute uint32_t capacity; SBlockID id; int16_t hasVarCol; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a75046d06d..0692db6a25 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -973,7 +973,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { taosSort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn); int64_t p1 = taosGetTimestampUs(); - uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows); + uDebug("blockDataSort easy cost:%" PRId64 ", rows:%" PRId64 "\n", p1 - p0, pDataBlock->info.rows); return TSDB_CODE_SUCCESS; } else { // var data type @@ -1739,14 +1739,14 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int64_t tbUid = pBlock->info.id.uid; int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int16_t hasVarCol = pBlock->info.hasVarCol; - int32_t rows = pBlock->info.rows; + int64_t rows = pBlock->info.rows; int32_t sz = taosArrayGetSize(pBlock->pDataBlock); int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, tbUid); tlen += taosEncodeFixedI16(buf, numOfCols); tlen += taosEncodeFixedI16(buf, hasVarCol); - tlen += taosEncodeFixedI32(buf, rows); + tlen += taosEncodeFixedI64(buf, rows); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); @@ -1777,7 +1777,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { buf = taosDecodeFixedU64(buf, &pBlock->info.id.uid); buf = taosDecodeFixedI16(buf, &numOfCols); buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); - buf = taosDecodeFixedI32(buf, &pBlock->info.rows); + buf = taosDecodeFixedI64(buf, &pBlock->info.rows); buf = taosDecodeFixedI32(buf, &sz); pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); for (int32_t i = 0; i < sz; i++) { @@ -1981,7 +1981,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t len = 0; len += snprintf(dumpBuf + len, size - len, "===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 - "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", + "|rows:%" PRId64 "|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 3ed1b083e4..c75c675ec3 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -669,7 +669,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma #endif for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { SSDataBlock *output = taosArrayGetP(pResList, i); - smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, + smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRId64, output->info.id.uid, output->info.id.groupId, output->info.rows); STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 7f9563ae5f..90ff1f8a84 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -340,7 +340,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { continue; } ret->fetchType = FETCH_TYPE__DATA; - tqDebug("return data rows %d", ret->data.info.rows); + tqDebug("return data rows %" PRId64, ret->data.info.rows); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index bf47f25e15..15fdbe2212 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -286,14 +286,16 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) { SMemSkipListNode *pNode = pTbData->sl.pHead; int64_t rowsNum = 0; - while (true) { - pNode = SL_GET_NODE_FORWARD(pNode, 0); + while (NULL != pNode) { if (pNode == pTbData->sl.pTail) { return rowsNum; } rowsNum++; + pNode = SL_GET_NODE_FORWARD(pNode, 0); } + + return rowsNum; } void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8b6f318434..63d9b702fd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1773,7 +1773,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* setComposedBlockFlag(pReader, true); double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange:%" PRId64 + tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64 " - %" PRId64 ", uid:%" PRIu64 ", %s", pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid, pReader->idStr); @@ -2723,7 +2723,7 @@ _end: if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + " rows:%" PRId64 ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -2970,7 +2970,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + " rows:%" PRId64 ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -3060,7 +3060,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + " rows:%" PRId64 ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -4593,20 +4593,18 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; SSDataBlock* pBlock = pReader->pResBlock; - while (1) { - if (pReader->status.loadFromFile == false) { - break; - } - - code = readRowsCountFromFile(pReader); - if (code != TSDB_CODE_SUCCESS) { - return false; - } + if (pReader->status.loadFromFile == false) { + return false; + } - code = readRowsCountFromStt(pReader); - if (code != TSDB_CODE_SUCCESS) { - return false; - } + code = readRowsCountFromFile(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = readRowsCountFromStt(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; } code = readRowsCountFromMem(pReader); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 67174c3267..8ef2a2b584 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1201,7 +1201,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { blockDataEnsureCapacity(pBlock, pBlock->info.rows + pRow->numOfRows); - qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", + qDebug("datablock capacity not sufficient, expand to required:%" PRId64 ", current capacity:%d, %s", (pRow->numOfRows+pBlock->info.rows), pBlock->info.capacity, GET_TASKID(pTaskInfo)); // todo set the pOperator->resultInfo size @@ -1214,7 +1214,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS pBlock->info.rows += pRow->numOfRows; } - qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, + qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.id.groupId); pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, 0); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 4e2e105d14..c943270df9 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -271,7 +271,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { return NULL; } } - qDebug("set op close, exec %d, status %d rows %d", pTaskInfo->execModel, pOperator->status, + qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); setOperatorCompleted(pOperator); break; @@ -337,7 +337,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint. if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) { - qDebug("project return %d rows, status %d", pFinalRes->info.rows, pOperator->status); + qDebug("project return %" PRId64 " rows, status %d", pFinalRes->info.rows, pOperator->status); break; } } else { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 854a2c1aaf..12eaad8e2a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -311,14 +311,14 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca taosMemoryFreeClear(pBlock->pBlockAgg); if (*status == FUNC_DATA_REQUIRED_FILTEROUT) { - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d, uid:%" PRIu64, GET_TASKID(pTaskInfo), + qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); pCost->skipBlocks += 1; @@ -329,7 +329,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca loadSMA = true; // mark the operation of load sma; bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead - qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); tsdbReleaseDataBlock(pTableScanInfo->dataReader); @@ -349,7 +349,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca size_t size = taosArrayGetSize(pBlock->pDataBlock); bool keep = doFilterByBlockSMA(pOperator->exprSupp.pFilterInfo, pBlock->pBlockAgg, size, pBlockInfo->rows); if (!keep) { - qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; @@ -366,7 +366,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca // try to filter data block according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; tsdbReleaseDataBlock(pTableScanInfo->dataReader); @@ -397,7 +397,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms", + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", elapsed time:%.2f ms", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el); } else { qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el); @@ -1028,7 +1028,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } tsdbReaderClose(pReader); - qDebug("retrieve prev rows:%d, skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 + qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1621,7 +1621,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows, + qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion); pTaskInfo->streamInfo.returned = 1; return pResult; @@ -1658,7 +1658,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { setBlockIntoRes(pInfo, &ret.data, true); if (pInfo->pRes->info.rows > 0) { pOperator->status = OP_EXEC_RECV; - qDebug("queue scan log return %d rows", pInfo->pRes->info.rows); + qDebug("queue scan log return %" PRId64 " rows", pInfo->pRes->info.rows); return pInfo->pRes; } } else if (ret.fetchType == FETCH_TYPE__META) { @@ -1848,7 +1848,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pCreateTbRes, "recover createTbl"); return pInfo->pCreateTbRes; } - qDebug("stream recover scan get block, rows %d", pInfo->pRecoverRes->info.rows); + qDebug("stream recover scan get block, rows %" PRId64 , pInfo->pRecoverRes->info.rows); printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; } @@ -2077,7 +2077,7 @@ FETCH_NEXT_BLOCK: pOperator->resultInfo.totalRows += pBlockInfo->rows; // printDataBlock(pInfo->pRes, "stream scan"); - qDebug("scan rows: %d", pBlockInfo->rows); + qDebug("scan rows: %" PRId64 , pBlockInfo->rows); if (pBlockInfo->rows > 0) { return pInfo->pRes; } @@ -2822,7 +2822,7 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* } bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo); - qDebug("%s get sorted row block, rows:%d, limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, + qDebug("%s get sorted row block, rows:%" PRId64 ", limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, pInfo->limitInfo.numOfOutputRows); return (pResBlock->info.rows > 0) ? pResBlock : NULL; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 90c7fa10ca..cb0f1aa068 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -698,7 +698,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData pDataBlock->info.dataLoad = 1; } - qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId, + qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%" PRId64 , GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ec4049a3eb..92c7852dbc 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -199,7 +199,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { QW_ERR_JRET(code); } - QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); + QW_TASK_DLOG("data put into sink, rows:%" PRId64 ", continueExecTask:%d", pRes->info.rows, qcontinue); } if (numOfResBlock == 0 || (hasMore == false)) { From 4a085db970a7d332bdade580863fb1f77f45fa33 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 29 Mar 2023 11:10:15 +0800 Subject: [PATCH 052/176] fix: hotfix version compatibility --- source/client/src/clientMsgHandler.c | 4 ++-- source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/util/src/tversion.c | 19 ++++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 21590022b1..a0146cfa39 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -76,7 +76,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { goto End; } - if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 2)) != 0) { + if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 3)) != 0) { setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); goto End; @@ -506,7 +506,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true); } - if(code != 0){ + if (code != 0) { taosMemoryFree(pRes); } tFreeSShowVariablesRsp(&rsp); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 41dea50731..50e502f4ab 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -226,7 +226,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 2)) != 0) { + if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 3)) != 0) { terrno = code; goto _OVER; } diff --git a/source/util/src/tversion.c b/source/util/src/tversion.c index dc357c61a1..0f84f6bafa 100644 --- a/source/util/src/tversion.c +++ b/source/util/src/tversion.c @@ -26,8 +26,9 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { int32_t vnum[4] = {0}; int32_t len = strlen(vstr); char tmp[16] = {0}; + int32_t vpos = 0; - for (int32_t spos = 0, tpos = 0, vpos = 0; spos < len && vpos < 4; ++spos) { + for (int32_t spos = 0, tpos = 0; spos < len && vpos < 4; ++spos) { if (vstr[spos] != '.') { tmp[spos - tpos] = vstr[spos]; } else { @@ -38,6 +39,10 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { } } + if ('\0' != tmp[0] && vpos < 4) { + vnum[vpos] = atoi(tmp); + } + if (vnum[0] <= 0) { terrno = TSDB_CODE_INVALID_VERSION_STRING; return -1; @@ -66,16 +71,16 @@ int32_t taosCheckVersionCompatible(int32_t clientVer, int32_t serverVer, int32_t case 4: break; case 3: - clientVer %= 100; - serverVer %= 100; + clientVer /= 100; + serverVer /= 100; break; case 2: - clientVer %= 10000; - serverVer %= 10000; + clientVer /= 10000; + serverVer /= 10000; break; case 1: - clientVer %= 1000000; - serverVer %= 1000000; + clientVer /= 1000000; + serverVer /= 1000000; break; default: terrno = TSDB_CODE_INVALID_VERSION_NUMBER; From 5f68b65675d8509ccf79b213ab92ec5e6c39f74e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 11:20:37 +0800 Subject: [PATCH 053/176] fix: count elements number type issue --- include/libs/function/function.h | 2 +- source/libs/function/src/builtinsimpl.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index fb6ef26a8a..aa5c78195a 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -112,7 +112,7 @@ typedef struct SResultDataInfo { typedef struct SInputColumnInfoData { int32_t totalRows; // total rows in current columnar data int32_t startRowIndex; // handle started row index - int32_t numOfRows; // the number of rows needs to be handled + int64_t numOfRows; // the number of rows needs to be handled int32_t numOfInputCols; // PTS is not included bool colDataSMAIsSet; // if agg is set or not SColumnInfoData *pPTS; // primary timestamp column diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3c9f2fe8ca..313ab93f8d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -494,8 +494,8 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { - int32_t numOfElem = 0; +static int64_t getNumOfElems(SqlFunctionCtx* pCtx) { + int64_t numOfElem = 0; /* * 1. column data missing (schema modified) causes pInputCol->hasNull == true. pInput->colDataSMAIsSet == true; @@ -528,7 +528,7 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ int32_t countFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = 0; + int64_t numOfElem = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -555,7 +555,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { } int32_t countInvertFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = getNumOfElems(pCtx); + int64_t numOfElem = getNumOfElems(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); char* buf = GET_ROWCELL_INTERBUF(pResInfo); @@ -1929,7 +1929,7 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - qDebug("%s total %d rows will merge, %p", __FUNCTION__, pInput->numOfRows, pInfo->pHisto); + qDebug("%s total %" PRId64 " rows will merge, %p", __FUNCTION__, pInput->numOfRows, pInfo->pHisto); int32_t start = pInput->startRowIndex; for (int32_t i = start; i < start + pInput->numOfRows; ++i) { From 87b8b4c94697a046abb7343c3bb50ece20d087a9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 29 Mar 2023 11:33:45 +0800 Subject: [PATCH 054/176] fix: telnet_tcp and sml_json_all_types for main (#20661) * fix: taosbenchmark support same min/max for main * fix: enable telnet_tcp and sml_json_all_types * fix: tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json * fix: taosdump free tbname in loose mode for main --- cmake/taostools_CMakeLists.txt.in | 2 +- .../5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json | 2 +- tests/parallel_test/cases.task | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 40fa48b815..3f7a43ab2d 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2864326 + GIT_TAG e82b9fc SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json index 9e1241397f..e609fcfebd 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json @@ -16,7 +16,7 @@ "num_of_records_per_req": 10, "databases": [{ "dbinfo": { - "name": "db", + "name": "opentsdb_telnet", "drop": "yes" }, "super_tables": [{ diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c2f8e75e79..4d6b5ff05e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1103,9 +1103,9 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py -#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R -#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R #docs-examples test ,,n,docs-examples-test,bash python.sh From 2714ce2af830e260449963dbbd3d181debdc629d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 13:22:12 +0800 Subject: [PATCH 055/176] fix: rows count issue --- source/dnode/vnode/src/tsdb/tsdbRead.c | 77 ++++++++++++-------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 63d9b702fd..3e41bd0d11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3087,7 +3087,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } -static int32_t doSumBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { +static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { int64_t st = taosGetTimestampUs(); LRUHandle* handle = NULL; int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); @@ -3134,32 +3134,7 @@ _end: } -static int32_t readRowsCountFromFile(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - - while (1) { - bool hasNext = false; - int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); - if (code) { - return code; - } - - if (!hasNext) { // no data files on disk - break; - } - - code = doSumBlockRows(pReader, pReader->pFileReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - pReader->status.loadFromFile = false; - - return code; -} - -static int32_t readRowsCountFromStt(STsdbReader* pReader) { +static int32_t doSumSttBlockRows(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; SSttBlockLoadInfo* pBlockLoadInfo = NULL; @@ -3167,17 +3142,12 @@ static int32_t readRowsCountFromStt(STsdbReader* pReader) { for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file pBlockLoadInfo = &pLastBlockReader->pInfo[i]; - if (!pLastBlockReader->pInfo[i].sttBlockLoaded) { - pLastBlockReader->pInfo[i].sttBlockLoaded = true; - - code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); - if (code) { - return code; - } + code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); + if (code) { + return code; } size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); - if (size >= 1) { SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); @@ -3214,6 +3184,36 @@ static int32_t readRowsCountFromStt(STsdbReader* pReader) { return code; } +static int32_t readRowsCountFromFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + while (1) { + bool hasNext = false; + int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); + if (code) { + return code; + } + + if (!hasNext) { // no data files on disk + break; + } + + code = doSumFileBlockRows(pReader, pReader->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doSumSttBlockRows(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pReader->status.loadFromFile = false; + + return code; +} + static int32_t readRowsCountFromMem(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; int64_t memNum = 0, imemNum = 0; @@ -4597,12 +4597,7 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { return false; } - code = readRowsCountFromFile(pReader); - if (code != TSDB_CODE_SUCCESS) { - return false; - } - - code = readRowsCountFromStt(pReader); + code = readRowsCountFromFiles(pReader); if (code != TSDB_CODE_SUCCESS) { return false; } From 8cf8ac84d71e06f674147f99e0ff03acc6c2ef9d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 29 Mar 2023 14:19:15 +0800 Subject: [PATCH 056/176] fix(tsdb/read): use correct scheme for mem & imem merging --- source/dnode/vnode/src/tsdb/tsdbRead.c | 37 +++++++++++++------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 3d8b2a4031..1906ae1abd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -177,15 +177,15 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; // the newest version schema -// STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times - SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema - SDataFReader* pFileReader; // the file reader - SDelFReader* pDelFReader; // the del file reader - SArray* pDelIdx; // del file block index; - SBlockInfoBuf blockInfoBuf; - int32_t step; - STsdbReader* innerReader[2]; + STSchema* pSchema; // the newest version schema + // STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times + SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; + SBlockInfoBuf blockInfoBuf; + int32_t step; + STsdbReader* innerReader[2]; }; static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); @@ -1862,11 +1862,11 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* void** p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion)); if (p != NULL) { - return *(STSchema**) p; + return *(STSchema**)p; } STSchema* ptr = NULL; - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -3674,8 +3674,9 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, piRow, pSchema); + int32_t code = tsdbRowMergerInit2(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4000,7 +4001,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { } static void freeSchemaFunc(void* param) { - void* p = *(void**) param; + void* p = *(void**)param; taosMemoryFree(p); } @@ -4082,7 +4083,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); if (pReader->pSchemaMap == NULL) { - tsdbError("failed init schema hash for reader", pReader->idStr); + tsdbError("failed init schema hash for reader %s", pReader->idStr); code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -4710,7 +4711,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { return TSDB_CODE_SUCCESS; } - SReaderStatus* pStatus = &pReader->status; + SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; pReader->order = pCond->order; @@ -4731,9 +4732,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { resetDataBlockIterator(pBlockIter, pReader->order); resetTableListIndex(&pReader->status); - bool asc = ASCENDING_TRAVERSE(pReader->order); - int32_t step = asc? 1:-1; - int64_t ts = asc? pReader->window.skey - 1 : pReader->window.ekey + 1; + bool asc = ASCENDING_TRAVERSE(pReader->order); + int32_t step = asc ? 1 : -1; + int64_t ts = asc ? pReader->window.skey - 1 : pReader->window.ekey + 1; resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step); int32_t code = 0; From c1452851e39cf323865895cb72226a3f26ee4ec2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 29 Mar 2023 14:39:37 +0800 Subject: [PATCH 057/176] fix: error in determining whether last(t.*) is a selection function --- source/libs/parser/src/parTranslater.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 726790443e..7fb5b0ba40 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1634,13 +1634,15 @@ static bool isTableStar(SNode* pNode) { (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); } +static bool isStarParam(SNode* pNode) { return isStar(pNode) || isTableStar(pNode); } + static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsMultiResFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } if (SQL_CLAUSE_SELECT != pCxt->currClause) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - if (isStar(pPara) || isTableStar(pPara)) { + if (isStarParam(pPara)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s(*) is only supported in SELECTed list", pFunc->functionName); } @@ -1654,7 +1656,7 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu static int32_t getMultiResFuncNum(SNodeList* pParameterList) { if (1 == LIST_LENGTH(pParameterList)) { - return isStar(nodesListGetNode(pParameterList, 0)) ? 2 : 1; + return isStarParam(nodesListGetNode(pParameterList, 0)) ? 2 : 1; } return LIST_LENGTH(pParameterList); } From 91c16f03446270779e64ab39d7bf4e93ed84e6ee Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 29 Mar 2023 15:34:55 +0800 Subject: [PATCH 058/176] tsdb/read: remove dup mem schema fetching --- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 38cb206877..0522869902 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3687,7 +3687,6 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); tsdbRowMergerAdd(&merge, pRow, pSchema); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); From 84eef542d5850f0a8a8852c81a998cf706a2dfda Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 15:45:18 +0800 Subject: [PATCH 059/176] fix: memory rows count issue --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 15fdbe2212..d0ff403bf7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -287,12 +287,12 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) { int64_t rowsNum = 0; while (NULL != pNode) { + pNode = SL_GET_NODE_FORWARD(pNode, 0); if (pNode == pTbData->sl.pTail) { return rowsNum; } rowsNum++; - pNode = SL_GET_NODE_FORWARD(pNode, 0); } return rowsNum; From 4f06728c1797a065badfda71b217f5ec4f1f83e8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 15:48:11 +0800 Subject: [PATCH 060/176] fix: restore sl level --- source/dnode/vnode/src/vnd/vnodeCfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 54d2eabfa4..c326c8bfac 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -29,7 +29,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI, .update = 1, .compression = 2, - .slLevel = 1, + .slLevel = 5, .days = 14400, .minRows = 100, .maxRows = 4096, From 8b7622b5e76fdf96a2f938649676abf3d78215c7 Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:51:49 +0800 Subject: [PATCH 061/176] Update tglobal.c --- source/common/src/tglobal.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e97523df3d..aeeec1d61c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1347,15 +1347,9 @@ void taosCleanupCfg() { } } -extern int32_t scanDebug; void taosCfgDynamicOptions(const char *option, const char *value) { if (strncasecmp(option, "debugFlag", 9) == 0) { int32_t flag = atoi(value); - if (1 == flag) { - scanDebug = 1; - } else { - scanDebug = 0; - } taosSetAllDebugFlag(flag, true); return; } From a90354cd928b3fbbc0b8ce07aba622ddbb88ab05 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 15:56:08 +0800 Subject: [PATCH 062/176] fix: restore debug code --- source/dnode/vnode/src/tsdb/tsdbRead.c | 39 +------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b9a06c787a..99e2e620e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3064,9 +3064,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - return code; - } code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3075,19 +3072,12 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // build composed data block code = buildComposedDataBlock(pReader); } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - return code; - } // data in memory that are earlier than current file block // rows in buffer should be less than the file block in asc, greater than file block in desc int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - return code; - } - // only return the rows in last block int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); ASSERT(tsLast >= pBlock->maxKey.ts); @@ -3437,12 +3427,6 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { initBlockDumpInfo(pReader, pBlockIter); } else { if (pReader->status.pCurrentFileset->nSttF > 0) { - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - pReader->pResBlock->info.rows = pReader->rowsNum; - pReader->rowsNum = 0; - return TSDB_CODE_SUCCESS; - } - // data blocks in current file are exhausted, let's try the next file now SBlockData* pBlockData = &pReader->status.fileBlockData; if (pBlockData->uid != 0) { @@ -3457,17 +3441,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { code = initForFirstBlockInFile(pReader, pBlockIter); // error happens or all the data files are completely checked - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - pReader->pResBlock->info.rows = pReader->rowsNum; - pReader->rowsNum = 0; - return TSDB_CODE_SUCCESS; - } - - if (pReader->status.loadFromFile == false) { + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { return code; } @@ -3481,17 +3455,6 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } code = doBuildDataBlock(pReader); - if (READ_MODE_COUNT_ONLY == pReader->readMode) { - if (false == pReader->status.composedDataBlock && pDumpInfo->allDumped) { - pReader->rowsNum += pReader->pResBlock->info.rows; - pReader->pResBlock->info.rows = 0; - continue; - } else if (pReader->pResBlock->info.rows == 0 && pReader->rowsNum > 0) { - pReader->pResBlock->info.rows = pReader->rowsNum; - pReader->rowsNum = 0; - return TSDB_CODE_SUCCESS; - } - } } if (code != TSDB_CODE_SUCCESS) { From 152fc5daab94f1b418d1f9fcf2ae24b59da1a191 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Mar 2023 16:36:35 +0800 Subject: [PATCH 063/176] fix(stream):fix the race condition when creating new tsables. --- source/dnode/vnode/src/tq/tqExec.c | 4 +- source/libs/executor/inc/executil.h | 1 + source/libs/executor/inc/executorimpl.h | 23 ++- source/libs/executor/src/executil.c | 17 +- source/libs/executor/src/executor.c | 212 +++++++++++++----------- source/libs/executor/src/executorimpl.c | 1 + source/libs/executor/src/scanoperator.c | 32 ++-- 7 files changed, 156 insertions(+), 134 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index f23b5f8526..18de4b75d0 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -68,14 +68,14 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs int32_t vgId = TD_VID(pTq->pVnode); if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return, consumer:0x%"PRIx64, pHandle->consumerId); + tqDebug("prepare scan failed, vgId:%d, consumer:0x%"PRIx64, vgId, pHandle->consumerId); if (pOffset->type == TMQ_OFFSET__LOG) { pRsp->rspOffset = *pOffset; return 0; } else { tqOffsetResetToLog(pOffset, pHandle->snapshotVer); if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return, consumer:0x%"PRIx64, pHandle->consumerId); + tqDebug("prepare scan failed, vgId:%d, consumer:0x%"PRIx64, vgId, pHandle->consumerId); pRsp->rspOffset = *pOffset; return 0; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f99c7de93d..9b8f034e44 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -107,6 +107,7 @@ int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t uint64_t tableListGetSize(const STableListInfo* pTableList); uint64_t tableListGetSuid(const STableListInfo* pTableList); STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index); +int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex); size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); void initResultRowInfo(SResultRowInfo* pResultRowInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 3f519568c4..c269367eb2 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -143,10 +143,7 @@ typedef struct { SQueryTableDataCond tableCond; int64_t fillHistoryVer1; int64_t fillHistoryVer2; - - // int8_t triggerSaved; - // int64_t deleteMarkSaved; - SStreamState* pState; + SStreamState* pState; } SStreamTaskInfo; typedef struct { @@ -168,15 +165,14 @@ typedef struct STaskStopInfo { } STaskStopInfo; struct SExecTaskInfo { - STaskIdInfo id; - uint32_t status; - STimeWindow window; - STaskCostInfo cost; - int64_t owner; // if it is in execution - int32_t code; - int32_t qbufQuota; // total available buffer (in KB) during execution query - - int64_t version; // used for stream to record wal version, why not move to sschemainfo + STaskIdInfo id; + uint32_t status; + STimeWindow window; + STaskCostInfo cost; + int64_t owner; // if it is in execution + int32_t code; + int32_t qbufQuota; // total available buffer (in KB) during execution query + int64_t version; // used for stream to record wal version, why not move to sschemainfo SStreamTaskInfo streamInfo; SSchemaInfo schemaInfo; STableListInfo* pTableInfoList; // this is a table list @@ -188,6 +184,7 @@ struct SExecTaskInfo { SLocalFetch localFetch; SArray* pResultBlockList; // result block list STaskStopInfo stopInfo; + SRWLatch lock; // secure the access of STableListInfo }; enum { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 953d614951..347a38247e 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -34,7 +34,7 @@ struct STableListInfo { int32_t numOfOuputGroups; // the data block will be generated one by one int32_t* groupOffset; // keep the offset value for each group in the tableList SArray* pTableList; - SHashObj* map; // speedup acquire the tableQueryInfo by table uid + SHashObj* map; // speedup acquire the tableQueryInfo by table uid uint64_t suid; }; @@ -1800,6 +1800,21 @@ STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) return taosArrayGet(pTableList->pTableList, index); } +int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex) { + int32_t numOfTables = taosArrayGetSize(pTableList->pTableList); + if (startIndex >= numOfTables) { + return -1; + } + + for (int32_t i = startIndex; i < numOfTables; ++i) { + STableKeyInfo* p = taosArrayGet(pTableList->pTableList, i); + if (p->uid == uid) { + return i; + } + } + return -1; +} + uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) { int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid)); ASSERT(pTableList->map != NULL && slot != NULL); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 85f17c0d53..5b71cdcac9 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -410,6 +410,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + taosWLockLatch(&pTaskInfo->lock); for (int32_t i = 0; i < numOfQualifiedTables; ++i) { uint64_t* uid = taosArrayGet(qa, i); @@ -424,6 +425,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(keyBuf); taosArrayDestroy(qa); + taosWUnLockLatch(&pTaskInfo->lock); return code; } } @@ -445,6 +447,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo tableListAddTableInfo(pTableListInfo, keyInfo.uid, keyInfo.groupId); } + taosWUnLockLatch(&pTaskInfo->lock); if (keyBuf != NULL) { taosMemoryFree(keyBuf); } @@ -452,7 +455,9 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo taosArrayDestroy(qa); } else { // remove the table id in current list qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); + taosWLockLatch(&pTaskInfo->lock); code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); + taosWUnLockLatch(&pTaskInfo->lock); } return code; @@ -1000,6 +1005,7 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { } return 0; } + bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; return pTaskInfo->streamInfo.recoverScanFinished; @@ -1080,8 +1086,11 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { } int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - SOperatorInfo* pOperator = pTaskInfo->pRoot; + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + const char* id = GET_TASKID(pTaskInfo); + pTaskInfo->streamInfo.prepareStatus = *pOffset; pTaskInfo->streamInfo.returned = 0; @@ -1095,20 +1104,23 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // TODO add more check if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if(pOperator->numOfDownstream != 1){ - qError("pOperator->numOfDownstream != 1:%d", pOperator->numOfDownstream); + qError("invalid operator, number of downstream:%d, %s", pOperator->numOfDownstream, id); return -1; } pOperator = pOperator->pDownstream[0]; } SStreamScanInfo* pInfo = pOperator->info; + STableScanInfo* pScanInfo = pInfo->pTableScanOp->info; + STableScanBase* pScanBaseInfo = &pScanInfo->base; + if (pOffset->type == TMQ_OFFSET__LOG) { - STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); - pTSInfo->base.dataReader = NULL; + tsdbReaderClose(pScanBaseInfo->dataReader); + pScanBaseInfo->dataReader = NULL; + // let's seek to the next version in wal file if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) { - qError("tqSeekVer failed ver:%" PRId64, pOffset->version + 1); + qError("tqSeekVer failed ver:%, %s" PRId64, pOffset->version + 1, id); return -1; } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { @@ -1117,120 +1129,120 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int64_t uid = pOffset->uid; int64_t ts = pOffset->ts; + // this value may be changed if new tables are created + taosRLockLatch(&pTaskInfo->lock); + int32_t numOfTables = tableListGetSize(pTableListInfo); + if (uid == 0) { - if (tableListGetSize(pTaskInfo->pTableInfoList) != 0) { - STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, 0); + if (numOfTables != 0) { + STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0); uid = pTableInfo->uid; ts = INT64_MIN; } else { - qError("uid == 0 and tablelist size is 0"); + taosRUnLockLatch(&pTaskInfo->lock); + qError("no table in table list, %s", id); return -1; } } - /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ - /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); - - qDebug("switch to table uid:%" PRId64 " ts:%" PRId64 "% "PRId64 " rows returned", uid, ts, pInfo->pTableScanOp->resultInfo.totalRows); pInfo->pTableScanOp->resultInfo.totalRows = 0; - bool found = false; - for (int32_t i = 0; i < numOfTables; i++) { - STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, i); - if (pTableInfo->uid == uid) { - found = true; - pTableScanInfo->currentTable = i; - break; - } - } + // start from current accessed position + int32_t index = tableListFind(pTableListInfo, uid, pScanInfo->currentTable); + taosRUnLockLatch(&pTaskInfo->lock); - // TODO after dropping table, table may not found - if(!found){ - qError("uid not found in tablelist %" PRId64, uid); + if (index >= 0) { + pScanInfo->currentTable = index; + } else { + qError("uid:%" PRIu64 " not found in table list, total:%d %s", uid, numOfTables, id); return -1; } - if (pTableScanInfo->base.dataReader == NULL) { - STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); - int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); - - if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num, - pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 || - pTableScanInfo->base.dataReader == NULL) { - qError("tsdbReaderOpen failed. uid:%" PRIi64, pOffset->uid); + STableKeyInfo keyInfo = {.uid = uid}; + if (pScanBaseInfo->dataReader == NULL) { + int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, + pScanInfo->pResBlock, &pScanBaseInfo->dataReader, NULL); + if (code != TSDB_CODE_SUCCESS) { + qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); + terrno = code; return -1; } + } else { + tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); + int64_t oldSkey = pScanBaseInfo->cond.twindows.skey; + + // let's start from the next ts that returned to consumer. + pScanBaseInfo->cond.twindows.skey = ts + 1; + tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); + + // restore the key value + pScanBaseInfo->cond.twindows.skey = oldSkey; + pScanInfo->scanTimes = 0; + + qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", + uid, ts, pScanInfo->currentTable, numOfTables, id); + } + } else { + qError("invalid pOffset->type:%d, %s", pOffset->type, id); + return -1; + } + + } else { // subType == TOPIC_SUB_TYPE__TABLE/DB + if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { + SStreamRawScanInfo* pInfo = pOperator->info; + SSnapContext* sContext = pInfo->sContext; + if (setForSnapShot(sContext, pOffset->uid) != 0) { + qError("setDataForSnapShot error. uid:%" PRIi64, pOffset->uid); + return -1; } - STableKeyInfo tki = {.uid = uid}; - tsdbSetTableList(pTableScanInfo->base.dataReader, &tki, 1); - int64_t oldSkey = pTableScanInfo->base.cond.twindows.skey; - pTableScanInfo->base.cond.twindows.skey = ts + 1; - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); - pTableScanInfo->base.cond.twindows.skey = oldSkey; - pTableScanInfo->scanTimes = 0; + SMetaTableInfo mtInfo = getUidfromSnapShot(sContext); + tsdbReaderClose(pInfo->dataReader); + pInfo->dataReader = NULL; - qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, - ts, pTableScanInfo->currentTable, numOfTables); - } else { - qError("invalid pOffset->type:%d", pOffset->type); - return -1; + cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); + tableListClear(pTableListInfo); + + if (mtInfo.uid == 0) { + return 0; // no data + } + + initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); + pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; + +// if (pTableListInfo == NULL) { +// pTableListInfo = tableListCreate(); +// } + + tableListAddTableInfo(pTableListInfo, mtInfo.uid, 0); + + STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); + int32_t size = tableListGetSize(pTableListInfo); + + tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL); + + cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); + strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); + tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); + pTaskInfo->streamInfo.schema = mtInfo.schema; + + qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64, mtInfo.uid, pOffset->ts); + } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { + SStreamRawScanInfo* pInfo = pOperator->info; + SSnapContext* sContext = pInfo->sContext; + if (setForSnapShot(sContext, pOffset->uid) != 0) { + qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); + return -1; + } + qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64, pOffset->uid, pOffset->ts); + } else if (pOffset->type == TMQ_OFFSET__LOG) { + SStreamRawScanInfo* pInfo = pOperator->info; + tsdbReaderClose(pInfo->dataReader); + pInfo->dataReader = NULL; + qDebug("tmqsnap qStreamPrepareScan snapshot log"); } - } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - SStreamRawScanInfo* pInfo = pOperator->info; - SSnapContext* sContext = pInfo->sContext; - if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setDataForSnapShot error. uid:%" PRIi64, pOffset->uid); - return -1; - } - - SMetaTableInfo mtInfo = getUidfromSnapShot(sContext); - tsdbReaderClose(pInfo->dataReader); - pInfo->dataReader = NULL; - - cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); - tableListClear(pTaskInfo->pTableInfoList); - - if (mtInfo.uid == 0) { - return 0; // no data - } - - initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); - pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; - - if (pTaskInfo->pTableInfoList == NULL) { - pTaskInfo->pTableInfoList = tableListCreate(); - } - - tableListAddTableInfo(pTaskInfo->pTableInfoList, mtInfo.uid, 0); - - STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); - int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); - - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL); - - cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); - strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); - tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); - pTaskInfo->streamInfo.schema = mtInfo.schema; - - qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64, mtInfo.uid, pOffset->ts); - } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { - SStreamRawScanInfo* pInfo = pOperator->info; - SSnapContext* sContext = pInfo->sContext; - if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); - return -1; - } - qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64, pOffset->uid, pOffset->ts); - } else if (pOffset->type == TMQ_OFFSET__LOG) { - SStreamRawScanInfo* pInfo = pOperator->info; - tsdbReaderClose(pInfo->dataReader); - pInfo->dataReader = NULL; - qDebug("tmqsnap qStreamPrepareScan snapshot log"); } + return 0; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 24a26d575a..3a6ab4463a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1989,6 +1989,7 @@ static SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, in pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); + taosInitRWLatch(&pTaskInfo->lock); pTaskInfo->id.vgId = vgId; pTaskInfo->id.queryId = queryId; pTaskInfo->id.str = buildTaskId(taskId, queryId); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5dff1abb97..b2197017df 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -766,8 +766,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable); tsdbSetTableList(pInfo->base.dataReader, pTableInfo, 1); - qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables, - pInfo->currentTable, pTaskInfo->id.str); + qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", pTableInfo->uid, numOfTables, + pInfo->currentTable, numOfTables, pTaskInfo->id.str); tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; @@ -1569,19 +1569,16 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; + const char* id = GET_TASKID(pTaskInfo); - qDebug("start to exec queue scan"); + qDebug("start to exec queue scan, %s", id); if (pTaskInfo->streamInfo.submit.msgStr != NULL) { - if (pInfo->tqReader->msg2.msgStr == NULL) { - /*pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq;*/ - /*const SSubmitReq* pSubmit = pInfo->tqReader->pMsg;*/ - /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ - /*void* msgStr = pTaskInfo->streamInfo.*/ + if (pInfo->tqReader->msg2.msgStr == NULL) { SPackedData submit = pTaskInfo->streamInfo.submit; if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { - qError("submit msg messed up when initing stream submit block %p", submit.msgStr); + qError("submit msg messed up when initing stream submit block %p, %s", submit.msgStr, id); pInfo->tqReader->msg2 = (SPackedData){0}; pInfo->tqReader->setMsg = 0; ASSERT(0); @@ -1615,18 +1612,20 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows, - pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion); + qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64" %s", pResult->info.rows, + pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion, id); pTaskInfo->streamInfo.returned = 1; return pResult; } else { + // no data has return already, try to extract data in the WAL if (!pTaskInfo->streamInfo.returned) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; tsdbReaderClose(pTSInfo->base.dataReader); - qDebug("3"); + pTSInfo->base.dataReader = NULL; tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); - qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); + + qDebug("queue scan tsdb over, switch to wal ver:%" PRId64 " %s", pTaskInfo->streamInfo.snapshotVer + 1, id); if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { tqOffsetResetToLog(&pTaskInfo->streamInfo.lastStatus, pTaskInfo->streamInfo.snapshotVer); return NULL; @@ -1643,7 +1642,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (tqNextBlock(pInfo->tqReader, &ret) < 0) { // if the end is reached, terrno is 0 if (terrno != 0) { - qError("failed to get next log block since %s", terrstr()); + qError("failed to get next log block since %s, %s", terrstr(), id); } } @@ -1658,9 +1657,6 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } else if (ret.fetchType == FETCH_TYPE__META) { qError("unexpected ret.fetchType:%d", ret.fetchType); continue; - // pTaskInfo->streamInfo.lastStatus = ret.offset; - // pTaskInfo->streamInfo.metaBlk = ret.meta; - // return NULL; } else if (ret.fetchType == FETCH_TYPE__NONE || (ret.fetchType == FETCH_TYPE__SEP && pOperator->status == OP_EXEC_RECV)) { pTaskInfo->streamInfo.lastStatus = ret.offset; @@ -1672,7 +1668,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } } } else { - qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.prepareStatus.type); + qError("unexpected streamInfo prepare type: %d %s", pTaskInfo->streamInfo.prepareStatus.type, id); return NULL; } } From f61732dca32c228f0167de961e80012b2a85ac10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Wed, 29 Mar 2023 16:56:53 +0800 Subject: [PATCH 064/176] test: refine query cases --- tests/system-test/2-query/columnLenUpdated.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py index d6940fde8b..ea01cd623c 100644 --- a/tests/system-test/2-query/columnLenUpdated.py +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -147,9 +147,9 @@ class TDTestCase: tdSql.checkData(1, 1, '55555') - tdSql.query("create table stb (ts timestamp, f1 int) tags (tg1 binary(2))") + tdSql.query("create table stb (ts timestamp, f1 int, f2 binary(2)) tags (tg1 binary(2))") tdSql.query("create table tb1 using stb tags('bb')") - tdSql.query("insert into tb1 values (now, 2)") + tdSql.query("insert into tb1 values (now, 2,'22')") tdSql.query("select count(*) from stb group by tg1") tdSql.checkData(0, 0, 1) @@ -163,13 +163,23 @@ class TDTestCase: if retCode != "TAOS_OK": tdLog.exit("taos -s fail") - keyDict['s'] = "\"insert into db1.tb2 values (now, 2)\"" + keyDict['s'] = "\"insert into db1.tb2 values (now, 2,'22')\"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"alter table db1.stb modify column f2 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"insert into db1.tb2 values (now, 3,'55555')\"" retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '') if retCode != "TAOS_OK": tdLog.exit("taos -s fail") tdSql.query("select count(*) from stb group by tg1") - tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 0, 2) tdSql.checkData(1, 0, 1) From d998912ce8b8cf6b36838a71ff75b12cde2b06f0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Mar 2023 17:11:41 +0800 Subject: [PATCH 065/176] fix(query): fix syntax error. --- source/libs/executor/src/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 5b71cdcac9..6c4a4bbe88 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1120,7 +1120,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // let's seek to the next version in wal file if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) { - qError("tqSeekVer failed ver:%, %s" PRId64, pOffset->version + 1, id); + qError("tqSeekVer failed ver:%"PRId64", %s" PRId64, pOffset->version + 1, id); return -1; } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { From 9d9ae749d1d5d22367bab18daa15a4194f60961f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 29 Mar 2023 17:27:04 +0800 Subject: [PATCH 066/176] opti:tmq logic & fix CI cases --- source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/tq/tqExec.c | 1 - source/libs/executor/src/executor.c | 5 ++-- source/libs/executor/src/scanoperator.c | 4 ++-- tests/system-test/7-tmq/tmqDelete-1ctb.py | 28 ++++++++++------------- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f3abd00779..9035960f19 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -395,6 +395,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand return -1; } + // offset set to previous version when init tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index e67f986c3e..87c762dd03 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -65,7 +65,6 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; - int32_t vgId = TD_VID(pTq->pVnode); if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { tqError("prepare scan failed, return"); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 297a64aeb5..3cb11815b2 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1077,6 +1077,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; + // if pOffset equal to current offset, means continue consume if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } @@ -1097,7 +1098,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; tsdbReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; - // let's seek to the next version in wal file + // set version to read for wal is next, so +1 if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) { qError("tqSeekVer failed ver:%" PRId64, pOffset->version + 1); return -1; @@ -1119,8 +1120,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } } - /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ - /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2be6fd87ca..7488f8cf4c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1617,18 +1617,18 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; tsdbReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pTaskInfo->streamInfo.snapshotVer); qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { return NULL; } + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pTaskInfo->streamInfo.snapshotVer); } if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { while (1) { SFetchRet ret = {0}; tqNextBlock(pInfo->tqReader, &ret); - tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); //curVersion move to next, so currentOffset = curVersion - 1 if (ret.fetchType == FETCH_TYPE__DATA) { qDebug("doQueueScan get data from log %d rows, version:%" PRId64, pInfo->pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index 69d2f5e347..16aa402a6d 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -53,7 +53,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -237,10 +237,10 @@ class TDTestCase: if self.snapshot == 0: consumerId = 2 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4 + 3/4)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) elif self.snapshot == 1: consumerId = 3 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4)) topicList = topicFromStb1 ifcheckdata = 1 @@ -270,7 +270,7 @@ class TDTestCase: if totalConsumeRows != expectrowcnt: tdLog.exit("tmq consume rows error with snapshot = 0!") elif self.snapshot == 1: - if totalConsumeRows != totalRowsFromQuery: + if totalConsumeRows != expectrowcnt: tdLog.exit("tmq consume rows error with snapshot = 1!") # tmqCom.checkFileContent(consumerId, queryString) @@ -323,7 +323,7 @@ class TDTestCase: if self.snapshot == 0: consumerId = 4 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1/4 + 3/4)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) elif self.snapshot == 1: consumerId = 5 expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) @@ -369,11 +369,7 @@ class TDTestCase: tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt)) if self.snapshot == 0: - # If data writing is completed before consumer get snapshot, will consume 7500 from wal; - # If data writing has not started before consumer get snapshot, will consume 10000 from wal; - minRows = int(expectrowcnt * (1 - 1/4)) # 7500 - tdLog.info("consume rows should be between %d and %d, "%(minRows, expectrowcnt)) - if not ((totalConsumeRows >= minRows) and (totalConsumeRows <= expectrowcnt)): + if (totalConsumeRows != expectrowcnt): tdLog.exit("tmq consume rows error with snapshot = 0!") elif self.snapshot == 1: tdLog.info("consume rows should be between %d and %d, "%(totalRowsFromQuery, expectrowcnt)) @@ -494,7 +490,7 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 4 end ...... ") def run(self): - # tdSql.prepare() + tdSql.prepare() tdLog.printNoPrefix("=============================================") tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") self.snapshot = 0 @@ -520,11 +516,11 @@ class TDTestCase: self.prepareTestEnv() self.tmqCase3() - tdLog.printNoPrefix("=============================================") - tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") - self.snapshot = 0 - self.prepareTestEnv() - self.tmqCase4() + # tdLog.printNoPrefix("=============================================") + # tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") + # self.snapshot = 0 + # self.prepareTestEnv() + # self.tmqCase4() tdLog.printNoPrefix("====================================================================") tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") self.snapshot = 1 From 6a1150301eae3fa6c76784e99a0b6814a233654e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 17:30:20 +0800 Subject: [PATCH 067/176] fix: column rows size --- source/dnode/vnode/src/tq/tqScan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 1a166d326f..43e214137f 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -114,7 +114,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs pRsp->blockNum++; - tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%d, total blocks:%d", vgId, pHandle->consumerId, + tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%" PRId64 ", total blocks:%d", vgId, pHandle->consumerId, pDataBlock->info.rows, pRsp->blockNum); if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { From b25b6a8333c9280b4a616a4b1d4dae348667f304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Wed, 29 Mar 2023 18:16:46 +0800 Subject: [PATCH 068/176] test: refine query cases --- tests/parallel_test/cases.task | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 260d47032a..a50503e24b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -14,8 +14,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 @@ -24,7 +25,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py From 3810e57c1542c4deb332d6c20cafda120a7c99ea Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 19:10:10 +0800 Subject: [PATCH 069/176] fix: tag copy issue --- source/libs/executor/src/executil.c | 4 ++++ tests/parallel_test/cases.task | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 953d614951..0081ffcb0b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -571,6 +571,10 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis memcpy(pStart, data, len); pStart += len; } else if (IS_VAR_DATA_TYPE(pValue->info.type)) { + if (varDataTLen(data) > pValue->info.bytes) { + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + goto end; + } memcpy(pStart, data, varDataTLen(data)); pStart += varDataTLen(data); } else { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a50503e24b..ab7540ee59 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -14,9 +14,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 @@ -25,7 +25,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py From 56c98d77684049c00481330460992b53a73cd3ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Mar 2023 19:35:04 +0800 Subject: [PATCH 070/176] refactor: do some internal refactor. --- include/libs/executor/executor.h | 3 ++- source/dnode/vnode/src/tq/tq.c | 7 +++--- source/dnode/vnode/src/tq/tqMeta.c | 10 ++++---- source/libs/executor/inc/executorimpl.h | 8 ++----- source/libs/executor/src/executor.c | 24 +++++++------------ source/libs/executor/src/executorimpl.c | 4 ++-- tests/system-test/2-query/columnLenUpdated.py | 2 +- 7 files changed, 23 insertions(+), 35 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 19a6407b77..8d63306a88 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -78,7 +78,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v * @param SReadHandle * @return */ -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, SSchemaWrapper** pSchema); +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); /** * set the task Id, usually used by message queue process @@ -89,6 +89,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); + /** * Set multiple input data blocks for the stream scan. * @param tinfo diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4db53c1627..484a0559a8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -799,7 +799,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle tqHandle = {0}; pHandle = &tqHandle; - /*taosInitRWLatch(&pExec->lock);*/ uint64_t oldConsumerId = pHandle->consumerId; memcpy(pHandle->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN); @@ -834,7 +833,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg req.qmsg = NULL; pHandle->execHandle.task = - qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, &pHandle->execHandle.numOfCols, NULL); + qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, &pHandle->execHandle.numOfCols, req.newConsumerId); void* scanner = NULL; qExtractStreamScanner(pHandle->execHandle.task, &scanner); pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); @@ -847,7 +846,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext**)(&handle.sContext)); - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, NULL); + pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); pHandle->execHandle.execTb.suid = req.suid; @@ -865,7 +864,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext**)(&handle.sContext)); - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, NULL); + pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); } taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index b6bca1e4ca..cb4be7a539 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -307,7 +307,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { handle.execHandle.task = - qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, NULL); +` qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0); if (handle.execHandle.task == NULL) { tqError("cannot create exec task for %s", handle.subKey); code = -1; @@ -332,7 +332,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, NULL); + handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); @@ -341,7 +341,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid); for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid); + tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); tqReaderSetTbUidList(handle.execHandle.pExecReader, tbUidList); @@ -349,9 +349,9 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, NULL); + handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } - tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, TD_VID(pTq->pVnode)); + tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c269367eb2..0cb9955056 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -482,12 +482,6 @@ typedef struct SStreamScanInfo { } SStreamScanInfo; typedef struct { - // int8_t subType; - // bool withMeta; - // int64_t suid; - // int64_t snapVersion; - // void *metaInfo; - // void *dataInfo; SVnode* vnode; SSDataBlock pRes; // result SSDataBlock STsdbReader* dataReader; @@ -690,6 +684,8 @@ typedef struct SStreamFillOperatorInfo { #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) +SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, char* dbFName); + SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain); int32_t optrDummyOpenFn(SOperatorInfo* pOperator); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6c4a4bbe88..06813f0148 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -242,30 +242,28 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, return code; } -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, SSchemaWrapper** pSchema) { - if (msg == NULL) { - // create raw scan - SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id) { + if (msg == NULL) { // create raw scan + SExecTaskInfo* pTaskInfo = doCreateExecTaskInfo(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, ""); if (NULL == pTaskInfo) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - - pTaskInfo->cost.created = taosGetTimestampUs(); - pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE; pTaskInfo->pRoot = createRawScanOperatorInfo(pReaderHandle, pTaskInfo); if (NULL == pTaskInfo->pRoot) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pTaskInfo); return NULL; } + + qDebug("create raw scan task info completed, vgId:%d, %s", vgId, GET_TASKID(pTaskInfo)); return pTaskInfo; } struct SSubplan* pPlan = NULL; - int32_t code = qStringToSubplan(msg, &pPlan); + + int32_t code = qStringToSubplan(msg, &pPlan); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -292,9 +290,6 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 } } - if (pSchema) { - *pSchema = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw); - } return pTaskInfo; } @@ -1138,6 +1133,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0); uid = pTableInfo->uid; ts = INT64_MIN; + pScanInfo->currentTable = 0; } else { taosRUnLockLatch(&pTaskInfo->lock); qError("no table in table list, %s", id); @@ -1210,10 +1206,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; -// if (pTableListInfo == NULL) { -// pTableListInfo = tableListCreate(); -// } - tableListAddTableInfo(pTableListInfo, mtInfo.uid, 0); STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3a6ab4463a..c2a068e1cb 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1974,7 +1974,7 @@ char* buildTaskId(uint64_t taskId, uint64_t queryId) { return p; } -static SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, char* dbFName) { +SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, char* dbFName) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); if (pTaskInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1982,6 +1982,7 @@ static SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, in } setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); + pTaskInfo->cost.created = taosGetTimestampUs(); pTaskInfo->schemaInfo.dbname = taosStrdup(dbFName); pTaskInfo->execModel = model; @@ -2465,7 +2466,6 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand goto _complete; } - (*pTaskInfo)->cost.created = taosGetTimestampUs(); return TSDB_CODE_SUCCESS; _complete: diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py index d6940fde8b..e4555b4e79 100644 --- a/tests/system-test/2-query/columnLenUpdated.py +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -83,7 +83,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) From c8ad465a0d18e2aa68e93d179325c7732ecd57be Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Mar 2023 19:40:34 +0800 Subject: [PATCH 071/176] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tqMeta.c | 2 +- source/libs/executor/src/executor.c | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index cb4be7a539..a273f2edec 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -307,7 +307,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { handle.execHandle.task = -` qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0); + qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0); if (handle.execHandle.task == NULL) { tqError("cannot create exec task for %s", handle.subKey); code = -1; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 06813f0148..1b71a41586 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1098,7 +1098,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // TODO add more check if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - if(pOperator->numOfDownstream != 1){ + if (pOperator->numOfDownstream != 1) { qError("invalid operator, number of downstream:%d, %s", pOperator->numOfDownstream, id); return -1; } @@ -1115,7 +1115,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // let's seek to the next version in wal file if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) { - qError("tqSeekVer failed ver:%"PRId64", %s" PRId64, pOffset->version + 1, id); + qError("tqSeekVer failed ver:%"PRId64", %s", pOffset->version + 1, id); return -1; } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { @@ -1123,6 +1123,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // those data are from the snapshot in tsdb, besides the data in the wal file. int64_t uid = pOffset->uid; int64_t ts = pOffset->ts; + int32_t index = 0; // this value may be changed if new tables are created taosRLockLatch(&pTaskInfo->lock); @@ -1144,7 +1145,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pInfo->pTableScanOp->resultInfo.totalRows = 0; // start from current accessed position - int32_t index = tableListFind(pTableListInfo, uid, pScanInfo->currentTable); + index = tableListFind(pTableListInfo, uid, pScanInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); if (index >= 0) { @@ -1183,12 +1184,12 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT return -1; } - } else { // subType == TOPIC_SUB_TYPE__TABLE/DB + } else { // subType == TOPIC_SUB_TYPE__TABLE/TOPIC_SUB_TYPE__DB if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setDataForSnapShot error. uid:%" PRIi64, pOffset->uid); + qError("setDataForSnapShot error. uid:%" PRId64" , %s", pOffset->uid, id); return -1; } @@ -1218,7 +1219,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); pTaskInfo->streamInfo.schema = mtInfo.schema; - qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64, mtInfo.uid, pOffset->ts); + qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64" %s", mtInfo.uid, pOffset->ts, id); } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; @@ -1226,12 +1227,12 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); return -1; } - qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64, pOffset->uid, pOffset->ts); + qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64 " %s", pOffset->uid, pOffset->ts, id); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; tsdbReaderClose(pInfo->dataReader); pInfo->dataReader = NULL; - qDebug("tmqsnap qStreamPrepareScan snapshot log"); + qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } } From 84d75c2ec09da7454ba83ab8d945ff46616f3126 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 29 Mar 2023 19:51:13 +0800 Subject: [PATCH 072/176] fix: join after interval --- source/libs/executor/src/timewindowoperator.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b01143841c..70febcaf6a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -128,8 +128,9 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn if (end >= 0) { forwardRows = end; - if (pData[end + pos] == ekey) { + while (pData[end + pos] == ekey) { forwardRows += 1; + ++pos; } } } else { @@ -137,8 +138,9 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn if (end >= 0) { forwardRows = end; - if (pData[end + pos] == ekey) { + while (pData[end + pos] == ekey) { forwardRows += 1; + ++pos; } } // int32_t end = searchFn((char*)pData, pos + 1, ekey, order); From d9b3c638c3a2a7c5e299793faa1c57b8c3c9ced4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 29 Mar 2023 21:03:39 +0800 Subject: [PATCH 073/176] fix:add assert for debug --- source/common/src/tdataformat.c | 3 +++ source/libs/parser/src/parInsertUtil.c | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 34808aa389..9632750a18 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2453,6 +2453,9 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; } else { + if(ASSERT(varDataTLen(data + offset) <= bytes)){ + uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes); + } code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset), varDataLen(data + offset)); } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 132a3b2618..bbe36e0c80 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -656,6 +656,9 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } else { pStart += colLength[j]; } + if(ASSERT(pCol->nVal == numOfRows)){ + uError("tFields is null, pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); + } } }else{ for (int i = 0; i < numFields; i++) { @@ -684,6 +687,9 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } else { pStart += colLength[i]; } + if(ASSERT(pCol->nVal == numOfRows)){ + uError("tFields is not null, pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); + } boundInfo->pColIndex[j] = -1; break; } From c1b4f941997c039d0bfa7de110551c3c689843ca Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 29 Mar 2023 22:12:13 +0800 Subject: [PATCH 074/176] fix:add assert for debug --- source/common/src/tdataformat.c | 2 ++ source/libs/parser/src/parInsertUtil.c | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 9632750a18..d6ab974c6c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2455,6 +2455,8 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt } else { if(ASSERT(varDataTLen(data + offset) <= bytes)){ uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes); + code = TSDB_CODE_INVALID_PARA; + goto _exit; } code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset), varDataLen(data + offset)); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index bbe36e0c80..132a3b2618 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -656,9 +656,6 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } else { pStart += colLength[j]; } - if(ASSERT(pCol->nVal == numOfRows)){ - uError("tFields is null, pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); - } } }else{ for (int i = 0; i < numFields; i++) { @@ -687,9 +684,6 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } else { pStart += colLength[i]; } - if(ASSERT(pCol->nVal == numOfRows)){ - uError("tFields is not null, pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); - } boundInfo->pColIndex[j] = -1; break; } From 5f4cb41e288740e69a8f69d139af8e11400d5d66 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Mar 2023 23:45:41 +0800 Subject: [PATCH 075/176] fix(tmq): set correct start offset value. --- source/client/src/clientTmq.c | 12 ++++++++---- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/libs/executor/src/executor.c | 18 +++++++++++------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 111ca28cdc..a16ddce0aa 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -535,10 +535,14 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN atomic_add_fetch_32(&pParamSet->totalRspNum, 1); SEp* pEp = GET_ACTIVE_EP(&pVg->epSet); - tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%" PRId64 " prev:%" PRId64 - ", ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64, - tmq->consumerId, pOffset->subKey, pVg->vgId, pOffset->val.version, pVg->committedOffset.version, pEp->fqdn, - pEp->port, index + 1, totalVgroups, pMsgSendInfo->requestId); + char offsetBuf[80] = {0}; + tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffset->val); + + char commitBuf[80] = {0}; + tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->committedOffset); + tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%s prev:%s, ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64, + tmq->consumerId, pOffset->subKey, pVg->vgId, offsetBuf, commitBuf, pEp->fqdn, pEp->port, index + 1, + totalVgroups, pMsgSendInfo->requestId); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 484a0559a8..d4bdd633e9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -399,8 +399,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand char formatBuf[80]; tFormatOffset(formatBuf, 80, pOffsetVal); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue.", - consumerId, pHandle->subKey, vgId, formatBuf); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%"PRIx64, + consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); return 0; } else { // no poll occurs in this vnode for this topic, let's seek to the right offset value. diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 00695e14f4..85a62c4dd1 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -92,7 +92,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; - tqDebug("vgId:%d, tmq task start to execute, consumer:0x%" PRIx64, vgId, pHandle->consumerId); + tqDebug("vgId:%d, tmq task start to execute, consumer:0x%"PRIx64, vgId, pHandle->consumerId); code = qExecTask(task, &pDataBlock, &ts); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1b71a41586..60463bad5f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1156,6 +1156,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } STableKeyInfo keyInfo = {.uid = uid}; + int64_t oldSkey = pScanBaseInfo->cond.twindows.skey; + + // let's start from the next ts that returned to consumer. + pScanBaseInfo->cond.twindows.skey = ts + 1; + if (pScanBaseInfo->dataReader == NULL) { int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, pScanInfo->pResBlock, &pScanBaseInfo->dataReader, NULL); @@ -1164,21 +1169,20 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT terrno = code; return -1; } + + qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts %" PRId64 " table index:%d, total:%d, %s", + uid, ts, pScanInfo->currentTable, numOfTables, id); } else { tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); - int64_t oldSkey = pScanBaseInfo->cond.twindows.skey; - - // let's start from the next ts that returned to consumer. - pScanBaseInfo->cond.twindows.skey = ts + 1; tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); - - // restore the key value - pScanBaseInfo->cond.twindows.skey = oldSkey; pScanInfo->scanTimes = 0; qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", uid, ts, pScanInfo->currentTable, numOfTables, id); } + + // restore the key value + pScanBaseInfo->cond.twindows.skey = oldSkey; } else { qError("invalid pOffset->type:%d, %s", pOffset->type, id); return -1; From 06aa7d3750c74193711797bffcf3445598beb325 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 30 Mar 2023 08:54:00 +0800 Subject: [PATCH 076/176] fix: add test case for interval after join --- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/join_interval.sim | 42 +++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/script/tsim/query/join_interval.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 260d47032a..cf2d62a2c3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -868,6 +868,7 @@ ,,y,script,./test.sh -f tsim/query/session.sim ,,y,script,./test.sh -f tsim/query/udf.sim ,,y,script,./test.sh -f tsim/query/udf_with_const.sim +,,y,script,./test.sh -f tsim/query/join_interval.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/query/event.sim diff --git a/tests/script/tsim/query/join_interval.sim b/tests/script/tsim/query/join_interval.sim new file mode 100644 index 0000000000..14994a5cc1 --- /dev/null +++ b/tests/script/tsim/query/join_interval.sim @@ -0,0 +1,42 @@ + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c udf -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step create databases +sql create database d1 +sql create database d2 +sql create table d1.t1(ts timestamp, i int) tags(t int); +sql create table d2.t1(ts timestamp, i int); +sql insert into d1.t11 using d1.t1 tags(1) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4) +sql insert into d1.t12 using d1.t1 tags(2) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4) +sql insert into d1.t13 using d1.t1 tags(3) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4) + +sql insert into d2.t1 values(1500000000000,0)(1500000000001,1)(1500000000002,2) + +sql select _wstart,_wend,count((a.ts)),count(b.ts) from d1.t1 a, d2.t1 b where a.ts is not null and a.ts = b.ts interval(1a) ; +if $data02 != 3 then + return -1 +endi + +if $data03 != 3 then + return -1 +endi + +if $data12 != 3 then + return -1 +endi + +if $data13 != 3 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +if $data23 != 3 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT From f93efc8538f4c4eeebc8d5b0ae93492776b0ac7a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 09:21:34 +0800 Subject: [PATCH 077/176] fix:telemetry.py failed in windows --- tests/system-test/0-others/telemetry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py index bc5d276faa..c62e3c2487 100644 --- a/tests/system-test/0-others/telemetry.py +++ b/tests/system-test/0-others/telemetry.py @@ -181,7 +181,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - vgroups = "30" + vgroups = "8" sql = "create database db3 vgroups " + vgroups tdSql.query(sql) From a5c8713517f751a0999527c0d08499308d9cf8a4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 09:40:16 +0800 Subject: [PATCH 078/176] opti:disable set enable.heartbeat.background --- source/client/src/clientTmq.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 111ca28cdc..9e585f2957 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -320,15 +320,16 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "enable.heartbeat.background") == 0) { - if (strcasecmp(value, "true") == 0) { - conf->hbBgEnable = true; - return TMQ_CONF_OK; - } else if (strcasecmp(value, "false") == 0) { - conf->hbBgEnable = false; - return TMQ_CONF_OK; - } else { +// if (strcasecmp(value, "true") == 0) { +// conf->hbBgEnable = true; +// return TMQ_CONF_OK; +// } else if (strcasecmp(value, "false") == 0) { +// conf->hbBgEnable = false; +// return TMQ_CONF_OK; +// } else { + tscError("the default value of enable.heartbeat.background is true, can not be seted"); return TMQ_CONF_INVALID; - } +// } } if (strcasecmp(key, "td.connect.ip") == 0) { From 347c08bfeac0fe28c28adf46e300785e81d81ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:17:52 +0800 Subject: [PATCH 079/176] test: refine query cases --- tests/parallel_test/cases.task | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ab7540ee59..fcece56816 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -10,22 +10,21 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py @@ -47,6 +46,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 From 2e8e5a98e9125e3729c4956266c19289b3019470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:18:05 +0800 Subject: [PATCH 080/176] test: refine query cases --- tests/system-test/2-query/nestedQuery.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index 6557aad05f..1b843defce 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -6144,7 +6144,7 @@ class TDTestCase: startTime = time.time() - self.function_before_26() + #self.function_before_26() self.math_nest(['UNIQUE']) self.math_nest(['MODE']) @@ -6157,9 +6157,9 @@ class TDTestCase: # self.math_nest(['MAVG']) # self.math_nest(['HYPERLOGLOG']) # self.math_nest(['TAIL']) - # self.math_nest(['CSUM']) - # self.math_nest(['statecount','stateduration']) - # self.math_nest(['HISTOGRAM']) + self.math_nest(['CSUM']) + self.math_nest(['statecount','stateduration']) + self.math_nest(['HISTOGRAM']) # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER']) # self.str_nest(['LENGTH','CHAR_LENGTH']) From 01ae6da8a45c2d2e1af2fecb1d2d9d33e687e062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:18:22 +0800 Subject: [PATCH 081/176] test: refine query cases --- tests/system-test/2-query/nestedQuery_math.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/nestedQuery_math.py b/tests/system-test/2-query/nestedQuery_math.py index 2d0bbcb352..3e37a2c15e 100755 --- a/tests/system-test/2-query/nestedQuery_math.py +++ b/tests/system-test/2-query/nestedQuery_math.py @@ -34,9 +34,9 @@ class TDTestCase(TDTestCase): self.math_nest(['MAVG']) self.math_nest(['HYPERLOGLOG']) self.math_nest(['TAIL']) - self.math_nest(['CSUM']) - self.math_nest(['statecount','stateduration']) - self.math_nest(['HISTOGRAM']) + # self.math_nest(['CSUM']) + # self.math_nest(['statecount','stateduration']) + # self.math_nest(['HISTOGRAM']) # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER']) # self.str_nest(['LENGTH','CHAR_LENGTH']) From 8825b8931441684d7ee070629ea4df0babd52df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:18:36 +0800 Subject: [PATCH 082/176] test: refine query cases --- tests/system-test/2-query/out_of_order.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/system-test/2-query/out_of_order.py b/tests/system-test/2-query/out_of_order.py index 148b89fc58..47a9cc3c11 100644 --- a/tests/system-test/2-query/out_of_order.py +++ b/tests/system-test/2-query/out_of_order.py @@ -162,19 +162,18 @@ class TDTestCase: sql = "select count(*) from (select distinct(tbname) from %s.meters)" %dbname tdSql.query(sql) - num = tdSql.getData(0,0) + # 目前不需要了 + # num = tdSql.getData(0,0) - for i in range(0,num): - sql1 = "select count(*) from %s.d%d" %(dbname,i) - tdSql.query(sql1) - sql1_result = tdSql.getData(0,0) - tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + # for i in range(0,num): + # sql1 = "select count(*) from %s.d%d" %(dbname,i) + # tdSql.query(sql1) + # sql1_result = tdSql.getData(0,0) + # tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + def check_out_of_order(self,dbname,tables,per_table_num,order,replica): self.run_benchmark(dbname,tables,per_table_num,order,replica) - print("sleep 10 seconds") - #time.sleep(10) - print("sleep 10 seconds finish") self.run_sql(dbname) From cf99c2e69dc207fd80e81dee5cf0872b7739004a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:18:53 +0800 Subject: [PATCH 083/176] test: refine query cases --- tests/system-test/2-query/nestedQuery_26.py | 76 +++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 tests/system-test/2-query/nestedQuery_26.py diff --git a/tests/system-test/2-query/nestedQuery_26.py b/tests/system-test/2-query/nestedQuery_26.py new file mode 100755 index 0000000000..9d5f31d1e0 --- /dev/null +++ b/tests/system-test/2-query/nestedQuery_26.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.cases import tdCases +from .nestedQuery import * + +class TDTestCase(TDTestCase): + + + def run(self): + tdSql.prepare() + + startTime = time.time() + + self.function_before_26() + + # self.math_nest(['UNIQUE']) + # self.math_nest(['MODE']) + # self.math_nest(['SAMPLE']) + + # self.math_nest(['ABS','SQRT']) + # self.math_nest(['SIN','COS','TAN','ASIN','ACOS','ATAN']) + # self.math_nest(['POW','LOG']) + # self.math_nest(['FLOOR','CEIL','ROUND']) + # self.math_nest(['MAVG']) + # self.math_nest(['HYPERLOGLOG']) + # self.math_nest(['TAIL']) + # self.math_nest(['CSUM']) + # self.math_nest(['statecount','stateduration']) + # self.math_nest(['HISTOGRAM']) + + # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER']) + # self.str_nest(['LENGTH','CHAR_LENGTH']) + # self.str_nest(['SUBSTR']) + # self.str_nest(['CONCAT']) + # self.str_nest(['CONCAT_WS']) + # self.time_nest(['CAST']) #放到time里起来弄 + # self.time_nest(['CAST_1']) + # self.time_nest(['CAST_2']) + # self.time_nest(['CAST_3']) + # self.time_nest(['CAST_4']) + + + + # self.time_nest(['NOW','TODAY']) + # self.time_nest(['TIMEZONE']) + # self.time_nest(['TIMETRUNCATE']) + # self.time_nest(['TO_ISO8601']) + # self.time_nest(['TO_UNIXTIMESTAMP']) + # self.time_nest(['ELAPSED']) + #self.time_nest(['TIMEDIFF_1']) + #self.time_nest(['TIMEDIFF_2']) + + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 2aeda10174be4cfb674009683710db6b93220843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:22:17 +0800 Subject: [PATCH 084/176] test: refine query cases --- tests/system-test/2-query/out_of_order.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/out_of_order.py b/tests/system-test/2-query/out_of_order.py index 47a9cc3c11..ff2b71193b 100644 --- a/tests/system-test/2-query/out_of_order.py +++ b/tests/system-test/2-query/out_of_order.py @@ -181,7 +181,7 @@ class TDTestCase: startTime = time.time() #self.check_out_of_order('db1',10,random.randint(10000,50000),random.randint(1,10),1) - self.check_out_of_order('db1',random.randint(50,200),random.randint(10000,20000),random.randint(1,5),1) + self.check_out_of_order('db1',random.randint(50,100),random.randint(10000,20000),random.randint(1,5),1) # self.check_out_of_order('db2',random.randint(50,200),random.randint(10000,50000),random.randint(5,50),1) From 07171a01b4da0962919ddf8dca19e31181655520 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 10:52:39 +0800 Subject: [PATCH 085/176] fix:taosdMonitor.py failed in windows --- tests/system-test/0-others/taosdMonitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 944ff52d5b..195f1ba5bc 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -292,7 +292,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - vgroups = "30" + vgroups = "8" sql = "create database db3 vgroups " + vgroups tdSql.query(sql) sql = "create table db3.stb (ts timestamp, f int) tags (t int)" From f30cb8e52722b1b2edf74bb7e3bab69d2cb5aa73 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 30 Mar 2023 10:58:33 +0800 Subject: [PATCH 086/176] fix: when the set operator statement is used for subquery, the outer layer filtering fails. --- source/libs/planner/src/planPhysiCreater.c | 13 +++++++++++-- source/libs/planner/src/planSpliter.c | 3 ++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e9a2efaac7..e2c2e4c655 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1089,9 +1089,15 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic pExchange->srcStartGroupId = pExchangeLogicNode->srcStartGroupId; pExchange->srcEndGroupId = pExchangeLogicNode->srcEndGroupId; pExchange->seqRecvData = pExchangeLogicNode->seqRecvData; - *pPhyNode = (SPhysiNode*)pExchange; - return TSDB_CODE_SUCCESS; + int32_t code = setConditionsSlotId(pCxt, (const SLogicNode*)pExchangeLogicNode, (SPhysiNode*)pExchange); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pExchange; + } else { + nodesDestroyNode((SNode*)pExchange); + } + + return code; } static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, @@ -1119,6 +1125,9 @@ static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExc if (TSDB_CODE_SUCCESS == code) { code = addDataBlockSlots(pCxt, pScan->pScanCols, pScan->node.pOutputDataBlockDesc); } + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pExchangeLogicNode, (SPhysiNode*)pScan); + } if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pScan; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 361cf33d58..4a3d689f04 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1365,7 +1365,8 @@ static int32_t unAllSplCreateExchangeNode(SSplitContext* pCxt, int32_t startGrou pExchange->srcEndGroupId = pCxt->groupId - 1; pExchange->node.precision = pProject->node.precision; pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets); - if (NULL == pExchange->node.pTargets) { + pExchange->node.pConditions = nodesCloneNode(pProject->node.pConditions); + if (NULL == pExchange->node.pTargets || NULL == pExchange->node.pConditions) { return TSDB_CODE_OUT_OF_MEMORY; } TSWAP(pExchange->node.pLimit, pProject->node.pLimit); From 8f3c41fc709d38a9735d644fa8cfd1759fcfc65a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 30 Mar 2023 11:03:40 +0800 Subject: [PATCH 087/176] fix: add more column update cases --- tests/system-test/2-query/columnLenUpdated.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py index ea01cd623c..3b87cbe22a 100644 --- a/tests/system-test/2-query/columnLenUpdated.py +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -147,6 +147,30 @@ class TDTestCase: tdSql.checkData(1, 1, '55555') + + keyDict['s'] = "\"alter table db1.tba add column f2 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select * from tba order by ts") + tdSql.query("select * from tba order by ts") + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + + + + + keyDict['s'] = "\"alter table db1.tba add column f3 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select f3 from tba order by ts") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + + tdSql.query("create table stb (ts timestamp, f1 int, f2 binary(2)) tags (tg1 binary(2))") tdSql.query("create table tb1 using stb tags('bb')") tdSql.query("insert into tb1 values (now, 2,'22')") From b7e6e4197dbc125093759eaea41afa6095803e6c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 11:18:35 +0800 Subject: [PATCH 088/176] opti:disable set enable.heartbeat.background --- docs/en/07-develop/07-tmq.mdx | 3 --- docs/examples/go/sub/main.go | 1 - docs/zh/07-develop/07-tmq.mdx | 3 --- tests/system-test/0-others/tmqBasic.json | 1 - utils/test/c/tmq_taosx_ci.c | 1 - 5 files changed, 9 deletions(-) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index c85109d3c5..dc0f07d3d5 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -293,7 +293,6 @@ You configure the following parameters when creating a consumer: | `auto.offset.reset` | enum | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | | `enable.auto.commit` | boolean | Commit automatically | Specify `true` or `false`. | | `auto.commit.interval.ms` | integer | Interval for automatic commits, in milliseconds | -| `enable.heartbeat.background` | boolean | Backend heartbeat; if enabled, the consumer does not go offline even if it has not polled for a long time | | | `experimental.snapshot.enable` | boolean | Specify whether to consume messages from the WAL or from TSBS | | | `msg.with.table.name` | boolean | Specify whether to deserialize table names from messages | @@ -368,7 +367,6 @@ conf := &tmq.ConfigMap{ "td.connect.port": "6030", "client.id": "test_tmq_c", "enable.auto.commit": "false", - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "true", } @@ -418,7 +416,6 @@ Python programs use the following parameters: | `auto.commit.interval.ms` | string | Interval for automatic commits, in milliseconds | | | `auto.offset.reset` | string | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | | `experimental.snapshot.enable` | string | Specify whether to consume messages from the WAL or from TSDB | Specify `true` or `false` | -| `enable.heartbeat.background` | string | Backend heartbeat; if enabled, the consumer does not go offline even if it has not polled for a long time | Specify `true` or `false` | diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go index 1f7218936f..334b8f290b 100644 --- a/docs/examples/go/sub/main.go +++ b/docs/examples/go/sub/main.go @@ -35,7 +35,6 @@ func main() { "td.connect.port": "6030", "client.id": "test_tmq_client", "enable.auto.commit": "false", - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "true", }) diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index fb171042d9..5771e5053a 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -291,7 +291,6 @@ CREATE TOPIC topic_name AS DATABASE db_name; | `auto.offset.reset` | enum | 消费组订阅的初始位置 | 可选：`earliest`(default), `latest`, `none` | | `enable.auto.commit` | boolean | 是否启用消费位点自动提交 | 合法值：`true`, `false`。 | | `auto.commit.interval.ms` | integer | 以毫秒为单位的消费记录自动提交消费位点时间间 | 默认 5000 m | -| `enable.heartbeat.background` | boolean | 启用后台心跳，启用后即使长时间不 poll 消息也不会造成离线 | 默认开启 | | `experimental.snapshot.enable` | boolean | 是否允许从 TSDB 消费数据 | 实验功能，默认关闭 | | `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句） | | @@ -366,7 +365,6 @@ conf := &tmq.ConfigMap{ "td.connect.port": "6030", "client.id": "test_tmq_c", "enable.auto.commit": "false", - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "true", } @@ -418,7 +416,6 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) | `auto.commit.interval.ms` | string | 以毫秒为单位的自动提交时间间隔 | 默认值：5000 ms | | `auto.offset.reset` | string | 消费组订阅的初始位置 | 可选：`earliest`(default), `latest`, `none` | | `experimental.snapshot.enable` | string | 是否允许从 TSDB 消费数据 | 合法值：`true`, `false` | -| `enable.heartbeat.background` | string | 启用后台心跳，启用后即使长时间不 poll 消息也不会造成离线 | 合法值：`true`, `false` | diff --git a/tests/system-test/0-others/tmqBasic.json b/tests/system-test/0-others/tmqBasic.json index 24e815708a..d716bff3ac 100644 --- a/tests/system-test/0-others/tmqBasic.json +++ b/tests/system-test/0-others/tmqBasic.json @@ -14,7 +14,6 @@ "auto.offset.reset": "earliest", "enable.auto.commit": "true", "auto.commit.interval.ms": 1000, - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "false", "topic_list": [ diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 1f25eae366..6661d6d14a 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -542,7 +542,6 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "enable.heartbeat.background", "true"); if (g_conf.snapShot) { tmq_conf_set(conf, "experimental.snapshot.enable", "true"); From e1d85ce0747f465d79f49ffd07ddee89de854f96 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Mar 2023 12:04:47 +0800 Subject: [PATCH 089/176] fix(tmq): fix error. --- source/libs/executor/src/executor.c | 18 ++++++++++-------- source/libs/executor/src/scanoperator.c | 3 ++- tests/system-test/7-tmq/subscribeDb.py | 4 ++-- .../7-tmq/tmqConsFromTsdb-mutilVg.py | 2 ++ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 60463bad5f..656ffda0ca 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1145,13 +1145,16 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pInfo->pTableScanOp->resultInfo.totalRows = 0; // start from current accessed position - index = tableListFind(pTableListInfo, uid, pScanInfo->currentTable); + // we cannot start from the pScanInfo->currentTable, since the commit offset may cause the rollback of the start + // position, let's find it from the beginning. + index = tableListFind(pTableListInfo, uid, 0); taosRUnLockLatch(&pTaskInfo->lock); if (index >= 0) { pScanInfo->currentTable = index; } else { - qError("uid:%" PRIu64 " not found in table list, total:%d %s", uid, numOfTables, id); + qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid, + numOfTables, pScanInfo->currentTable, id); return -1; } @@ -1160,25 +1163,24 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // let's start from the next ts that returned to consumer. pScanBaseInfo->cond.twindows.skey = ts + 1; + pScanInfo->scanTimes = 0; if (pScanBaseInfo->dataReader == NULL) { int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, - pScanInfo->pResBlock, &pScanBaseInfo->dataReader, NULL); + pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id); if (code != TSDB_CODE_SUCCESS) { qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); terrno = code; return -1; } - qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts %" PRId64 " table index:%d, total:%d, %s", - uid, ts, pScanInfo->currentTable, numOfTables, id); + qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } else { tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); - pScanInfo->scanTimes = 0; - qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", - uid, ts, pScanInfo->currentTable, numOfTables, id); + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } // restore the key value diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c13366c560..84317c825b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -779,13 +779,14 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { // if no data, switch to next table and continue scan pInfo->currentTable++; if (pInfo->currentTable >= numOfTables) { + qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); return NULL; } STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable); tsdbSetTableList(pInfo->base.dataReader, pTableInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", pTableInfo->uid, numOfTables, - pInfo->currentTable, numOfTables, pTaskInfo->id.str); + pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index 0fa9bcfbd4..c47c218891 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -13,11 +13,11 @@ from util.dnodes import * class TDTestCase: hostname = socket.gethostname() - #rpcDebugFlagVal = '143' + rpcDebugFlagVal = '143' #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} - #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 87832ac0ef..11fc7dbcc0 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -16,6 +16,8 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: + updatecfgDict = {"tsdbDebugFlag":135} + def __init__(self): self.vgroups = 4 self.ctbNum = 10 From 71d2620259e2313e0cf3446d21f182a48653096b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Mar 2023 12:58:52 +0800 Subject: [PATCH 090/176] other: update the test case. --- tests/system-test/7-tmq/subscribeDb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index c47c218891..41db943f83 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -13,11 +13,11 @@ from util.dnodes import * class TDTestCase: hostname = socket.gethostname() - rpcDebugFlagVal = '143' + # rpcDebugFlagVal = '143' #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} - updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + # updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): From 10979f6fc28199c6f6af9906599a8a140a8fef35 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 30 Mar 2023 13:34:35 +0800 Subject: [PATCH 091/176] fix: when the set operator statement is used for subquery, the outer layer filtering fails. --- source/libs/planner/src/planSpliter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 4a3d689f04..fd77261818 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1366,7 +1366,7 @@ static int32_t unAllSplCreateExchangeNode(SSplitContext* pCxt, int32_t startGrou pExchange->node.precision = pProject->node.precision; pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets); pExchange->node.pConditions = nodesCloneNode(pProject->node.pConditions); - if (NULL == pExchange->node.pTargets || NULL == pExchange->node.pConditions) { + if (NULL == pExchange->node.pTargets || (NULL != pProject->node.pConditions && NULL == pExchange->node.pConditions)) { return TSDB_CODE_OUT_OF_MEMORY; } TSWAP(pExchange->node.pLimit, pProject->node.pLimit); From 32c262acc41a0d0d2cf0268237b8c1e559ef3ad8 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Mar 2023 16:37:26 +0800 Subject: [PATCH 092/176] test: modify tmq test case --- tests/system-test/7-tmq/tmq3mnodeSwitch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index cdcdadbcbb..9ded8fc942 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -211,7 +211,7 @@ class TDTestCase: # init consume info, and start tmq_sim, then check consume result tdLog.info("insert consume info to consume processor") consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 # because taosd switch, may be consume duplication data topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 @@ -251,11 +251,12 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectRowsList[0] != resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.info("expect consume rows: %d should less/equal than act consume rows: %d"%(expectRowsList[0], resultList[0])) + if expectRowsList[0] <= resultList[0]: tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + if expectRowsList[0] == resultList[0]: + self.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): From 8c9580f93e6f19108c7ef9f7a4b801d401718ab5 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Mar 2023 16:38:43 +0800 Subject: [PATCH 093/176] test: modify tmq test case --- tests/system-test/7-tmq/tmq3mnodeSwitch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index 9ded8fc942..7c95f7a3db 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -252,7 +252,7 @@ class TDTestCase: resultList = tmqCom.selectConsumeResult(expectRows) tdLog.info("expect consume rows: %d should less/equal than act consume rows: %d"%(expectRowsList[0], resultList[0])) - if expectRowsList[0] <= resultList[0]: + if expectRowsList[0] > resultList[0]: tdLog.exit("0 tmq consume rows error!") if expectRowsList[0] == resultList[0]: From ae98ad43a5b4ea5daf3ddf63e642fc612ce4a336 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 30 Mar 2023 16:35:56 +0800 Subject: [PATCH 094/176] fix:state window return wrong block type --- source/libs/executor/src/scanoperator.c | 10 ++++ tests/script/tsim/stream/state1.sim | 66 +++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 84317c825b..1e5c6c2168 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1829,6 +1829,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pUpdateRes, "recover update"); return pInfo->pUpdateRes; } break; + case STREAM_SCAN_FROM_DELETE_DATA: { + generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; + printDataBlock(pInfo->pDeleteDataRes, "recover delete"); + return pInfo->pDeleteDataRes; + } break; case STREAM_SCAN_FROM_DATAREADER_RANGE: { SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); if (pSDB) { @@ -2021,6 +2030,7 @@ FETCH_NEXT_BLOCK: copyDataBlock(pInfo->pUpdateRes, pSup->pScanBlock); blockDataCleanup(pSup->pScanBlock); prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA; return pInfo->pUpdateRes; } diff --git a/tests/script/tsim/stream/state1.sim b/tests/script/tsim/stream/state1.sim index 2ae5739642..67e02c0890 100644 --- a/tests/script/tsim/stream/state1.sim +++ b/tests/script/tsim/stream/state1.sim @@ -4,6 +4,7 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +print step 1 print =============== create database sql create database test vgroups 4; sql select * from information_schema.ins_databases; @@ -33,8 +34,8 @@ if $loop_count == 10 then endi sql select * from streamt1; -print data00 data01 -print data10 data11 +print $data00 $data01 +print $data10 $data11 if $rows != 0 then print =====rows=$rows @@ -52,8 +53,8 @@ if $loop_count == 10 then endi sql select * from streamt1; -print data00 data01 -print data10 data11 +print $data00 $data01 +print $data10 $data11 if $rows != 1 then print =====rows=$rows @@ -92,9 +93,64 @@ endi sql select * from streamt1; if $rows != 2 then print =====rows=$rows - goto loop2 + goto loop3 endi +print step 1 over +print step 2 + +sql create database test2 vgroups 1; +sql use test2; +sql create table t1(ts timestamp, a int, b int , c int, d double); +print create stream streams2 trigger at_once watermark 1000s into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a) +sql create stream streams2 trigger at_once watermark 1000s into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791213010,1,2,3,1.1); + +$loop_count = 0 +loop4: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt2; +print $data00 $data01 +print $data10 $data11 + +if $rows != 1 then + print =====rows=$rows + goto loop4 +endi + +print insert into t1 values(1648791213005,2,2,3,1.1) +sql insert into t1 values(1648791213005,2,2,3,1.1); + +$loop_count = 0 +loop5: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print select * from streamt2 +sql select * from streamt2; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 + +if $rows != 3 then + print =====rows=$rows + goto loop5 +endi + +print step 2 over print state1 end From a3a3dc824355c4ff54a2f39d658181ffd4ebb3bb Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 30 Mar 2023 16:17:34 +0800 Subject: [PATCH 095/176] fix: add filter to exchange node for union-all subquery --- source/libs/executor/src/exchangeoperator.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 3bdecbe748..8caebd2a34 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -212,6 +212,11 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { return NULL; } + doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); + if (blockDataGetNumOfRows(pBlock) == 0) { + continue; + } + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; if (hasLimitOffsetInfo(pLimitInfo)) { int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); @@ -303,6 +308,11 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); + code = filterInitFromNode((SNode*)pExNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL); return pOperator; From b2ca68943c898c10b904e9c74abea1c9d7f077b7 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 30 Mar 2023 16:58:21 +0800 Subject: [PATCH 096/176] fix: add test case --- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/unionall_as_table.sim | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/script/tsim/query/unionall_as_table.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4aa751ae7e..49f14191b1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -874,6 +874,7 @@ ,,y,script,./test.sh -f tsim/query/udf.sim ,,y,script,./test.sh -f tsim/query/udf_with_const.sim ,,y,script,./test.sh -f tsim/query/join_interval.sim +,,y,script,./test.sh -f tsim/query/unionall_as_table.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/query/event.sim diff --git a/tests/script/tsim/query/unionall_as_table.sim b/tests/script/tsim/query/unionall_as_table.sim new file mode 100644 index 0000000000..dc3d2cbec4 --- /dev/null +++ b/tests/script/tsim/query/unionall_as_table.sim @@ -0,0 +1,28 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; +sql CREATE STABLE bw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50)); +sql CREATE STABLE aw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50)); +sql CREATE STABLE dw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50)); +sql insert into t1 using dw_yc_h_substation_mea tags('1234567890','testa','0021001','abc01') values(now,'2023-03-27','00:01:00',now,2.3,3.3,4.4,5.5); +sql insert into t2 using dw_yc_h_substation_mea tags('2234567890','testb','0022001','abc02') values(now,'2023-03-27','00:01:00',now,2.3,2.3,2.4,2.5); +sql insert into t3 using aw_yc_h_substation_mea tags('2234567890','testc','0023001','abc03') values(now,'2023-03-27','00:15:00',now,2.3,2.3,2.4,2.5); +sql insert into t4 using bw_yc_h_substation_mea tags('4234567890','testd','0021001','abc03') values(now,'2023-03-27','00:45:00',now,2.3,2.3,2.4,2.5); +sql insert into t5 using bw_yc_h_substation_mea tags('5234567890','testd','0021001','abc03') values(now,'2023-03-27','00:00:00',now,2.3,2.3,2.4,2.5); +sql select t.ts,t.id,t.name,t.create_date,t.create_time,t.datasource,t.sum_p_value from (select ts,id,name,create_date,create_time,datasource,sum_p_value from bw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45') union all select ts,id,name,create_date,create_time,datasource,sum_p_value from aw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45') union all select ts,id,name,create_date,create_time,datasource,sum_p_value from dw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45')) t where t.datasource='0021001' and t.id='4234567890' order by t.create_time; + +if $rows != 1 then + return -1 +endi +if $data01 != @4234567890@ then + return -1 +endi +if $data05 != @0021001@ then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From a8da9f31e70ec91cae7878501593f24d51bd4925 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 30 Mar 2023 18:04:32 +0800 Subject: [PATCH 097/176] fix: join query error --- source/libs/function/src/builtins.c | 4 ++-- source/libs/parser/src/parTranslater.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 629e846dae..1a039ebda6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2457,7 +2457,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "interp", .type = FUNCTION_TYPE_INTERP, .classification = FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC|FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateInterp, .getEnvFunc = getSelectivityFuncEnv, .initFunc = functionSetup, @@ -3278,7 +3278,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_irowts", .type = FUNCTION_TYPE_IROWTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC|FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7fb5b0ba40..f58a66af18 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -757,7 +757,7 @@ static bool isPrimaryKeyImpl(SNode* pExpr) { if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pFunc->funcType || FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType) { return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0)); - } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType) { + } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || FUNCTION_TYPE_IROWTS == pFunc->funcType) { return true; } } From 627cfb230bc0313f1d7799983ccee474b6e7ebfb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 30 Mar 2023 18:35:49 +0800 Subject: [PATCH 098/176] chore: update taos-tools d194dc9 for main (#20713) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 3f7a43ab2d..aef89a2d42 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG e82b9fc + GIT_TAG d194dc9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 7e30f8619c9acb87c7aac837fdfbf5e83a3d3f43 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Mar 2023 18:39:50 +0800 Subject: [PATCH 099/176] fix(query): fix group_key is processed in selectivity twice --- source/libs/function/src/builtinsimpl.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3c9f2fe8ca..f8714a53c0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -871,6 +871,12 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; + // group_key function has its own process function + // do not process there + if (fmIsGroupKeyFunc(pc->functionId)) { + continue; + } + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); if (nullList[j]) { colDataSetNULL(pDstCol, rowIndex); @@ -3091,6 +3097,12 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid for (int32_t i = 0; i < pSubsidiaryies->num; ++i) { SqlFunctionCtx* pc = pSubsidiaryies->pCtx[i]; + // group_key function has its own process function + // do not process there + if (fmIsGroupKeyFunc(pc->functionId)) { + continue; + } + SFunctParam* pFuncParam = &pc->pExpr->base.pParam[0]; int32_t srcSlotId = pFuncParam->pCol->slotId; From a9cc66ba8f32380f9f9a23382f125dcf62a56815 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 19:48:21 +0800 Subject: [PATCH 100/176] fix:int64 parse error in schemaless --- source/client/src/clientSml.c | 42 +++++++++++++---------------------- utils/test/c/sml_test.c | 4 ++-- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 1719759822..a4ecc3c3df 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -257,19 +257,14 @@ cleanup: kvVal->f = (float)result; #define SET_BIGINT \ - if (smlDoubleToInt64OverFlow(result)) { \ - errno = 0; \ - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ - if (errno == ERANGE) { \ - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_BIGINT; \ - kvVal->i = tmp; \ - return true; \ - } \ - kvVal->type = TSDB_DATA_TYPE_BIGINT; \ - kvVal->i = (int64_t)result; + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; #define SET_INT \ if (!IS_VALID_INT(result)) { \ @@ -288,19 +283,14 @@ cleanup: kvVal->i = result; #define SET_UBIGINT \ - if (result >= (double)UINT64_MAX || result < 0) { \ - errno = 0; \ - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ - if (errno == ERANGE || result < 0) { \ - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ - kvVal->u = tmp; \ - return true; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ - kvVal->u = result; + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; #define SET_UINT \ if (!IS_VALID_UINT(result)) { \ diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 873946121b..44fbd060be 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -850,7 +850,7 @@ int smlProcess_18784_Test() { taos_free_result(pRes); const char *sql[] = { - "disk,device=sdc inodes_used=176059i,total=1081101176832i 1661943960000000000", + "disk,device=sdc inodes_used=176059i,total=1076048383523889174i 1661943960000000000", "disk,device=sdc inodes_free=66932805i 1661943960000000000", }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); @@ -875,7 +875,7 @@ int smlProcess_18784_Test() { if (rowIndex == 0) { ASSERT(ts == 1661943960000); ASSERT(used == 176059); - ASSERT(total == 1081101176832); + ASSERT(total == 1076048383523889174); ASSERT(freed == 66932805); // ASSERT_EQ(latitude, 24.5208); // ASSERT_EQ(longitude, 28.09377); From 79784491139ea068bedfa9f9275ce796d5eb3f48 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Fri, 31 Mar 2023 09:41:20 +0800 Subject: [PATCH 101/176] enh(taosAdapter): TMQ parameter adjustment (#20710) --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 1c401ae80e..b2f335e1f7 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG d8059ff + GIT_TAG cb1e89c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 450c5b447c7a08e2326654a804b36710b33889f5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 10:35:13 +0800 Subject: [PATCH 102/176] fix(tmq): quit the scan asap. --- source/libs/executor/src/executor.c | 4 ++-- source/libs/executor/src/scanoperator.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 656ffda0ca..b8b58cc9a0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -755,8 +755,8 @@ int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); - setTaskKilled(pTaskInfo, rspCode); + qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo)); + setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); while(qTaskIsExecuting(pTaskInfo)) { taosMsleep(10); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 84317c825b..dfbd354bd3 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -772,7 +772,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { while (1) { SSDataBlock* result = doGroupedTableScan(pOperator); - if (result || (pOperator->status == OP_EXEC_DONE)) { + if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) { return result; } @@ -1666,6 +1666,8 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__LOG) { while (1) { SFetchRet ret = {0}; + terrno = 0; + if (tqNextBlock(pInfo->tqReader, &ret) < 0) { // if the end is reached, terrno is 0 if (terrno != 0) { From b0fb912da8e26e46d858e5923231be7ce1bb0086 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 11:48:52 +0800 Subject: [PATCH 103/176] fix(mnd): fix the invalid access of NULL ptr. --- source/dnode/mnode/impl/src/mndTrans.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 39b4252618..65ba76b21f 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1086,15 +1086,18 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { goto _OVER; } + pTrans->lastErrorNo = pRsp->code; + STransAction *pAction = taosArrayGet(pArray, action); if (pAction != NULL) { pAction->msgReceived = 1; pAction->errCode = pRsp->code; - pTrans->lastErrorNo = pRsp->code; + mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, + mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode); + } else { + mInfo("trans:%d, invalid action, index:%d, code:0x%x", transId, action, pRsp->code); } - mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, mndTransStr(pAction->stage), - action, pRsp->code, pAction->acceptableCode, pAction->retryCode); mndTransExecute(pMnode, pTrans, true); _OVER: From 373c128a7add2132e7b253ad71932c91f86548f6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 31 Mar 2023 14:16:39 +0800 Subject: [PATCH 104/176] fix:[TS-3038] coredump if nchar data not clear SColVal in schemaless --- source/libs/parser/src/parInsertSml.c | 2 ++ utils/test/c/sml_test.c | 3 +++ 2 files changed, 5 insertions(+) diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 106ee641af..0bcd777d2a 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -25,6 +25,8 @@ static void clearColValArray(SArray* pCols) { if (TSDB_DATA_TYPE_NCHAR == pCol->type) { taosMemoryFreeClear(pCol->value.pData); } + pCol->flag = CV_FLAG_NONE; + pCol->value.val = 0; } } diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 44fbd060be..b0cc6f749c 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -939,6 +939,9 @@ int sml_ts2164_Test() { // "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", "meters,location=la,groupid=ca current=11.8,voltage=221", "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "ts3038,location=l2a,groupid=ca current=L\"11.8\"", + "ts3038,location=l2a,groupid=ca voltage=L\"221\"", + "ts3038,location=l2a,groupid=ca phase=L\"221\"", // "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; From a88de7e18bddd517d04bf612fefaf9e8668157bb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 15:55:16 +0800 Subject: [PATCH 105/176] fix(tmq): build sync api based on async api. --- source/client/src/clientTmq.c | 499 +++++++++++++++++----------------- 1 file changed, 243 insertions(+), 256 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index a16ddce0aa..0a4e85068d 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -27,6 +27,8 @@ #define EMPTY_BLOCK_POLL_IDLE_DURATION 10 #define DEFAULT_AUTO_COMMIT_INTERVAL 5000 +typedef void (*__tmq_askep_fn_t)(tmq_t* pTmq, int32_t code, SDataBuf* pBuf, void* pParam); + struct SMqMgmt { int8_t inited; tmr_h timer; @@ -109,6 +111,10 @@ struct tmq_t { tsem_t rspSem; }; +typedef struct SAskEpInfo { + int32_t code; +} SAskEpInfo; + enum { TMQ_VG_STATUS__IDLE = 0, TMQ_VG_STATUS__WAIT, @@ -169,11 +175,10 @@ typedef struct { } SMqSubscribeCbParam; typedef struct { - int64_t refId; - int32_t epoch; - int32_t code; - int32_t async; - tsem_t rspSem; + int64_t refId; + int32_t epoch; + void* pParam; + __tmq_askep_fn_t pUserFn; } SMqAskEpCbParam; typedef struct { @@ -189,16 +194,13 @@ typedef struct { typedef struct { int64_t refId; int32_t epoch; - int8_t automatic; - int8_t async; int32_t waitingRspNum; int32_t totalRspNum; - int32_t rspErr; + int32_t code; tmq_commit_cb* userCb; /*SArray* successfulOffsets;*/ /*SArray* failedOffsets;*/ - void* userParam; - tsem_t rspSem; + void* userParam; } SMqCommitCbParamSet; typedef struct { @@ -209,12 +211,14 @@ typedef struct { tmq_t* pTmq; } SMqCommitCbParam; -static int32_t tmqAskEp(tmq_t* tmq, bool async); +static int32_t doAskEp(tmq_t* tmq); static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg); static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet, int32_t index, int32_t totalVgroups); -static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); +static void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); +static void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param); +static void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param); tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); @@ -443,7 +447,7 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { // taosMemoryFree(pBuf->pData); // taosMemoryFree(pBuf->pEpSet); // -// tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); +// commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); // return 0; // } // @@ -453,7 +457,7 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { taosMemoryFree(pBuf->pData); taosMemoryFree(pBuf->pEpSet); - tmqCommitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); + commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); return 0; } @@ -461,8 +465,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN int32_t index, int32_t totalVgroups) { STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); if (pOffset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pOffset->val = pVg->currentOffset; @@ -476,13 +479,13 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN int32_t code = 0; tEncodeSize(tEncodeSTqOffset, pOffset, len, code); if (code < 0) { - return -1; + return TSDB_CODE_INVALID_PARA; } void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); if (buf == NULL) { taosMemoryFree(pOffset); - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); @@ -499,7 +502,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN if (pParam == NULL) { taosMemoryFree(pOffset); taosMemoryFree(buf); - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pParam->params = pParamSet; @@ -515,7 +518,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN taosMemoryFree(pOffset); taosMemoryFree(buf); taosMemoryFree(pParam); - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pMsgSendInfo->msgInfo = (SDataBuf){ @@ -546,129 +549,117 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); - return 0; + + return TSDB_CODE_SUCCESS; } -static int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) { - char* topic; - int32_t vgId; - if (TD_RES_TMQ(msg)) { - SMqRspObj* pRspObj = (SMqRspObj*)msg; - topic = pRspObj->topic; +static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* pCommitFp, void* userParam) { + char* pTopicName = NULL; + int32_t vgId = 0; + int32_t code = 0; + + if (pRes == NULL || tmq == NULL) { + pCommitFp(tmq, TSDB_CODE_INVALID_PARA, userParam); + return; + } + + if (TD_RES_TMQ(pRes)) { + SMqRspObj* pRspObj = (SMqRspObj*)pRes; + pTopicName = pRspObj->topic; vgId = pRspObj->vgId; - } else if (TD_RES_TMQ_META(msg)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg; - topic = pMetaRspObj->topic; + } else if (TD_RES_TMQ_META(pRes)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; + pTopicName = pMetaRspObj->topic; vgId = pMetaRspObj->vgId; - } else if (TD_RES_TMQ_METADATA(msg)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)msg; - topic = pRspObj->topic; + } else if (TD_RES_TMQ_METADATA(pRes)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; + pTopicName = pRspObj->topic; vgId = pRspObj->vgId; } else { - return TSDB_CODE_TMQ_INVALID_MSG; + pCommitFp(tmq, TSDB_CODE_TMQ_INVALID_MSG, userParam); + return; } SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); + return; } pParamSet->refId = tmq->refId; pParamSet->epoch = tmq->epoch; - pParamSet->automatic = 0; - pParamSet->async = async; - pParamSet->userCb = userCb; + pParamSet->userCb = pCommitFp; pParamSet->userParam = userParam; - tsem_init(&pParamSet->rspSem, 0, 0); - int32_t code = -1; - - taosThreadMutexLock(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%" PRIx64 " user invoked commit offset for %d", tmq->consumerId, numOfTopics); - for (int32_t i = 0; i < numOfTopics; i++) { + tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); + + int32_t i = 0; + for (; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (strcmp(pTopic->topicName, topic) != 0) { - continue; - } - - int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - for (int32_t j = 0; j < numOfVgroups; j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->vgId != vgId) { - continue; - } - - if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { - if (doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups) < 0) { - tsem_destroy(&pParamSet->rspSem); - taosMemoryFree(pParamSet); - goto FAIL; - } - goto HANDLE_RSP; - } + if (strcmp(pTopic->topicName, pTopicName) == 0) { + break; } } -HANDLE_RSP: - if (pParamSet->totalRspNum == 0) { - tsem_destroy(&pParamSet->rspSem); + if (i == numOfTopics) { + tscWarn("consumer:0x" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId, pTopicName, + numOfTopics); taosMemoryFree(pParamSet); - taosThreadMutexUnlock(&tmq->lock); - return 0; + pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); + return; } - if (!async) { - taosThreadMutexUnlock(&tmq->lock); - tsem_wait(&pParamSet->rspSem); - code = pParamSet->rspErr; - tsem_destroy(&pParamSet->rspSem); + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + + int32_t j = 0; + int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); + for (j = 0; j < numOfVgroups; j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + if (pVg->vgId == vgId) { + break; + } + } + + if (j == numOfVgroups) { + tscWarn("consumer:0x" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s", tmq->consumerId, vgId, + numOfVgroups, pTopicName); taosMemoryFree(pParamSet); - return code; - } else { - code = 0; + pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); + return; } -FAIL: - taosThreadMutexUnlock(&tmq->lock); - if (code != 0 && async) { - userCb(tmq, code, userParam); - } + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { + code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups); - return 0; + // failed to commit, callback user function directly. + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pParamSet); + pCommitFp(tmq, code, userParam); + } + } else { // do not perform commit, callback user function directly. + taosMemoryFree(pParamSet); + pCommitFp(tmq, code, userParam); + } } -static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { - int32_t code = -1; - +static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - if (async) { - if (automatic) { - tmq->commitCb(tmq, code, tmq->commitCbUserParam); - } else { - userCb(tmq, code, userParam); - } - } - return -1; + pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); + return; } pParamSet->refId = tmq->refId; pParamSet->epoch = tmq->epoch; - - pParamSet->automatic = automatic; - pParamSet->async = async; - pParamSet->userCb = userCb; + pParamSet->userCb = pCommitFp; pParamSet->userParam = userParam; - tsem_init(&pParamSet->rspSem, 0, 0); // init as 1 to prevent concurrency issue pParamSet->waitingRspNum = 1; - taosThreadMutexLock(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); @@ -682,7 +673,7 @@ static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_comm SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { - code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups); + int32_t code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups); if (code != TSDB_CODE_SUCCESS) { tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 " failed, code:%s ordinal:%d/%d", tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->committedOffset.version, tstrerror(terrno), @@ -693,7 +684,7 @@ static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_comm // update the offset value. pVg->committedOffset = pVg->currentOffset; } else { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", + tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, not commit, current:%" PRId64 ", ordinal:%d/%d", tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset.version, j + 1, numOfVgroups); } } @@ -701,39 +692,16 @@ static int32_t doAutoCommit(tmq_t* tmq, int8_t automatic, int8_t async, tmq_comm tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, numOfTopics); - taosThreadMutexUnlock(&tmq->lock); // no request is sent if (pParamSet->totalRspNum == 0) { - tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); - return 0; + pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); + return; } // count down since waiting rsp num init as 1 - tmqCommitRspCountDown(pParamSet, tmq->consumerId, "", 0); - - if (!async) { - tsem_wait(&pParamSet->rspSem); - code = pParamSet->rspErr; - tsem_destroy(&pParamSet->rspSem); - taosMemoryFree(pParamSet); -#if 0 - taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); - taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); -#endif - } - - return code; -} - -static int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { - if (msg) { // user invoked commit - return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); - } else { // this for auto commit - return doAutoCommit(tmq, automatic, async, userCb, userParam); - } + commitRspCountDown(pParamSet, tmq->consumerId, "", 0); } static void generateTimedTask(int64_t refId, int32_t type) { @@ -855,7 +823,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { while (pTaskType != NULL) { if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { - tmqAskEp(pTmq, true); + asyncAskEp(pTmq, addToQueueCallbackFn, NULL); int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); *pRefId = pTmq->refId; @@ -863,12 +831,11 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); taosTmrReset(tmqAssignAskEpTask, 1000, pRefId, tmqMgmt.timer, &pTmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - tmqCommitInner(pTmq, NULL, 1, 1, pTmq->commitCb, pTmq->commitCbUserParam); - + asyncCommitAllOffsets(pTmq, pTmq->commitCb, pTmq->commitCbUserParam); int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); *pRefId = pTmq->refId; - tscDebug("consumer:0x%" PRIx64 " commit to vnode(s) in %.2fs", pTmq->consumerId, + tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, pRefId, tmqMgmt.timer, &pTmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { @@ -1213,7 +1180,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } int32_t retryCnt = 0; - while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { if (retryCnt++ > MAX_RETRY_COUNT) { goto FAIL; } @@ -1513,28 +1480,30 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { return set; } -static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { +int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; - int8_t async = pParam->async; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); if (tmq == NULL) { - if (!async) { - tsem_destroy(&pParam->rspSem); - } else { - taosMemoryFree(pParam); - } + terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; + pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); + taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); - terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - return -1; + taosMemoryFree(pParam); + return terrno; } - pParam->code = code; if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 ", get topic endpoint error, async:%d, code:%s", tmq->consumerId, pParam->async, - tstrerror(code)); - goto END; + tscError("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code)); + pParam->pUserFn(tmq, code, NULL, pParam->pParam); + + taosReleaseRef(tmqMgmt.rsetId, pParam->refId); + + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pParam); + return code; } // tmq's epoch is monotonically increase, @@ -1545,6 +1514,7 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { if (head->epoch <= epoch) { tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep", tmq->consumerId, head->epoch, epoch); + if (tmq->status == TMQ_CONSUMER_STATUS__RECOVER) { SMqAskEpRsp rsp; tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); @@ -1553,45 +1523,17 @@ static int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { tDeleteSMqAskEpRsp(&rsp); } - goto END; - } - - tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, - head->epoch, epoch); - - if (!async) { - SMqAskEpRsp rsp; - tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); - tmqUpdateEp(tmq, head->epoch, &rsp); - tDeleteSMqAskEpRsp(&rsp); } else { - SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0); - if (pWrapper == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; - goto END; - } - - pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; - pWrapper->epoch = head->epoch; - memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg); - - taosWriteQitem(tmq->mqueue, pWrapper); - tsem_post(&tmq->rspSem); + tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, + head->epoch, epoch); + pParam->pUserFn(tmq, code, pMsg, pParam->pParam); } -END: taosReleaseRef(tmqMgmt.rsetId, pParam->refId); - if (!async) { - tsem_post(&pParam->rspSem); - } else { - taosMemoryFree(pParam); - } - taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pData); + taosMemoryFree(pParam); return code; } @@ -1980,7 +1922,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { while (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { int32_t retryCnt = 0; - while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { if (retryCnt++ > 40) { return NULL; } @@ -2162,96 +2104,159 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { - tmqCommitInner(tmq, msg, 0, 1, cb, param); -} - -int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { - return tmqCommitInner(tmq, msg, 0, 0, NULL, NULL); -} - -int32_t tmqAskEp(tmq_t* tmq, bool async) { - int32_t code = TSDB_CODE_SUCCESS; -#if 0 - int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); - if (epStatus == 1) { - int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); - tscTrace("consumer:0x%" PRIx64 ", skip ask ep cnt %d", tmq->consumerId, epSkipCnt); - if (epSkipCnt < 5000) return 0; +void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* param) { + if (pRes == NULL) { // here needs to commit all offsets. + asyncCommitAllOffsets(tmq, cb, param); + } else { // only commit one offset + asyncCommitOffset(tmq, pRes, cb, param); } - atomic_store_32(&tmq->epSkipCnt, 0); -#endif +} +typedef struct SSyncCommitInfo { + tsem_t sem; + int32_t code; +} SSyncCommitInfo; + +static void commitCallBackFn(tmq_t *pTmq, int32_t code, void* param) { + SSyncCommitInfo* pInfo = (SSyncCommitInfo*) param; + pInfo->code = code; + tsem_post(&pInfo->sem); +} + +int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { + int32_t code = 0; + + SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); + tsem_init(&pInfo->sem, 0, 0); + pInfo->code = 0; + + if (pRes == NULL) { + asyncCommitOffset(tmq, pRes, commitCallBackFn, pInfo); + } else { + asyncCommitAllOffsets(tmq, commitCallBackFn, pInfo); + } + + tsem_wait(&pInfo->sem); + + code = pInfo->code; + taosMemoryFree(pInfo); + + tscDebug("consumer:0x%"PRIx64" sync commit done, code:%s", tmq->consumerId, tstrerror(code)); + return code; +} + +void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) { + SAskEpInfo* pInfo = param; + pInfo->code = code; + + if (code == TSDB_CODE_SUCCESS) { + SMqRspHead* head = pDataBuf->pData; + + SMqAskEpRsp rsp; + tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &rsp); + tmqUpdateEp(pTmq, head->epoch, &rsp); + tDeleteSMqAskEpRsp(&rsp); + } + + tsem_post(&pTmq->rspSem); +} + +void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) { + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return; + } + + SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0); + if (pWrapper == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + + SMqRspHead* head = pDataBuf->pData; + + pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; + pWrapper->epoch = head->epoch; + memcpy(&pWrapper->msg, pDataBuf->pData, sizeof(SMqRspHead)); + tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &pWrapper->msg); + + taosWriteQitem(pTmq->mqueue, pWrapper); +} + +int32_t doAskEp(tmq_t* pTmq) { + SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); + + asyncAskEp(pTmq, updateEpCallbackFn, pInfo); + tsem_wait(&pTmq->rspSem); + + int32_t code = pInfo->code; + taosMemoryFree(pInfo); + return code; +} + +void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param) { SMqAskEpReq req = {0}; - req.consumerId = tmq->consumerId; - req.epoch = tmq->epoch; - strcpy(req.cgroup, tmq->groupId); + req.consumerId = pTmq->consumerId; + req.epoch = pTmq->epoch; + strcpy(req.cgroup, pTmq->groupId); int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req); if (tlen < 0) { - tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", tmq->consumerId); - return -1; + tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", pTmq->consumerId); + askEpFn(pTmq, TSDB_CODE_INVALID_PARA, NULL, param); + return; } void* pReq = taosMemoryCalloc(1, tlen); if (pReq == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", tmq->consumerId, tlen); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", pTmq->consumerId, tlen); + askEpFn(pTmq, TSDB_CODE_OUT_OF_MEMORY, NULL, param); + return; } if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { - tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", tmq->consumerId, tlen); + tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", pTmq->consumerId, tlen); taosMemoryFree(pReq); - return -1; + + askEpFn(pTmq, TSDB_CODE_INVALID_PARA, NULL, param); + return; } SMqAskEpCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqAskEpCbParam)); if (pParam == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", tmq->consumerId); + tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId); taosMemoryFree(pReq); - return -1; + + askEpFn(pTmq, TSDB_CODE_OUT_OF_MEMORY, NULL, param); + return; } - pParam->refId = tmq->refId; - pParam->epoch = tmq->epoch; - pParam->async = async; - tsem_init(&pParam->rspSem, 0, 0); + pParam->refId = pTmq->refId; + pParam->epoch = pTmq->epoch; + pParam->pUserFn = askEpFn; + pParam->pParam = param; SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { - tsem_destroy(&pParam->rspSem); taosMemoryFree(pParam); taosMemoryFree(pReq); - return -1; + askEpFn(pTmq, TSDB_CODE_OUT_OF_MEMORY, NULL, param); + return; } - sendInfo->msgInfo = (SDataBuf){ - .pData = pReq, - .len = tlen, - .handle = NULL, - }; + sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = tlen, .handle = NULL }; sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->param = pParam; - sendInfo->fp = tmqAskEpCb; + sendInfo->fp = askEpCallbackFn; sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d, reqId:0x%" PRIx64, tmq->consumerId, async, - sendInfo->requestId); + SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); + tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - if (!async) { - tsem_wait(&pParam->rspSem); - code = pParam->code; - taosMemoryFree(pParam); - } - - return code; + asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); } int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg) { @@ -2263,38 +2268,20 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { - if (!pParamSet->async) { - tsem_destroy(&pParamSet->rspSem); - } taosMemoryFree(pParamSet); terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; return -1; } // if no more waiting rsp - if (pParamSet->async) { - // call async cb func - if (pParamSet->automatic && tmq->commitCb) { - tmq->commitCb(tmq, pParamSet->rspErr, tmq->commitCbUserParam); - } else if (!pParamSet->automatic && pParamSet->userCb) { // sem post - pParamSet->userCb(tmq, pParamSet->rspErr, pParamSet->userParam); - } - - taosMemoryFree(pParamSet); - } else { - tsem_post(&pParamSet->rspSem); - } - -#if 0 - taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); - taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); -#endif + pParamSet->userCb(tmq, pParamSet->code, pParamSet->userParam); + taosMemoryFree(pParamSet); taosReleaseRef(tmqMgmt.rsetId, refId); return 0; } -void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { +void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); if (waitingRspNum == 0) { tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, From f57899e0701a1ca7d6fab203a897d6b763497866 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 16:06:38 +0800 Subject: [PATCH 106/176] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index dfbd354bd3..468b535dc3 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1639,7 +1639,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64" %s", pResult->info.rows, + qDebug("queue scan tsdb return %d rows brange:%" PRId64 " - %" PRId64 " wal curVersion:%" PRId64" %s", pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion, id); pTaskInfo->streamInfo.returned = 1; return pResult; From 05b97544b072ed459d515c5a0f8b790cdb3f0bfb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 16:12:30 +0800 Subject: [PATCH 107/176] fix(tmq): fix the syntax error. --- source/client/src/clientTmq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 0a4e85068d..4558a57609 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -623,7 +623,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* p } if (j == numOfVgroups) { - tscWarn("consumer:0x" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s", tmq->consumerId, vgId, + tscWarn("consumer:0x%" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s", tmq->consumerId, vgId, numOfVgroups, pTopicName); taosMemoryFree(pParamSet); pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); From 41b2c2d8fd84a331e8d12ba83a5eed94a3933773 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 31 Mar 2023 16:16:14 +0800 Subject: [PATCH 108/176] fix: error in optimizing useless columns for multi-level set operators --- source/libs/parser/src/parCalcConst.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index a7f6d7cb3c..c25d0e7036 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -337,8 +337,14 @@ static SNodeList* getChildProjection(SNode* pStmt) { static void eraseSetOpChildProjection(SSetOperator* pSetOp, int32_t index) { SNodeList* pLeftProjs = getChildProjection(pSetOp->pLeft); nodesListErase(pLeftProjs, nodesListGetCell(pLeftProjs, index)); + if (QUERY_NODE_SET_OPERATOR == nodeType(pSetOp->pLeft)) { + eraseSetOpChildProjection((SSetOperator*)pSetOp->pLeft, index); + } SNodeList* pRightProjs = getChildProjection(pSetOp->pRight); nodesListErase(pRightProjs, nodesListGetCell(pRightProjs, index)); + if (QUERY_NODE_SET_OPERATOR == nodeType(pSetOp->pRight)) { + eraseSetOpChildProjection((SSetOperator*)pSetOp->pRight, index); + } } typedef struct SNotRefByOrderByCxt { From 32109e7322242ab1e0432dc53e8a903762f317f4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 16:21:46 +0800 Subject: [PATCH 109/176] fix(tmq): fix the syntax error. --- source/client/src/clientTmq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c521501ec6..86d4aee8ed 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -605,7 +605,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* p } if (i == numOfTopics) { - tscWarn("consumer:0x" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId, pTopicName, + tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId, pTopicName, numOfTopics); taosMemoryFree(pParamSet); pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); From ed21ef04201eebeda89a283d8bbdb00c352237cd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 17:06:33 +0800 Subject: [PATCH 110/176] fix(tmq): fix sync commit error. --- source/client/src/clientTmq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 86d4aee8ed..a627d5f190 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1315,7 +1315,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { char buf[80]; tFormatOffset(buf, 80, &pRspWrapper->dataRsp.rspOffset); - tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, + tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId); } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { SDecoder decoder; @@ -2132,9 +2132,9 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { pInfo->code = 0; if (pRes == NULL) { - asyncCommitOffset(tmq, pRes, commitCallBackFn, pInfo); - } else { asyncCommitAllOffsets(tmq, commitCallBackFn, pInfo); + } else { + asyncCommitOffset(tmq, pRes, commitCallBackFn, pInfo); } tsem_wait(&pInfo->sem); From 19e5d6372104647697264d00d6f511d06b7285fe Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 31 Mar 2023 18:26:46 +0800 Subject: [PATCH 111/176] fix: add job retry --- source/libs/qworker/src/qwDbg.c | 17 +++++++-- source/libs/scheduler/inc/schInt.h | 16 ++++++-- source/libs/scheduler/src/schJob.c | 53 +++++++++++++++++++++++++-- source/libs/scheduler/src/schRemote.c | 9 ++++- source/libs/scheduler/src/schTask.c | 31 +++++++++++++--- 5 files changed, 109 insertions(+), 17 deletions(-) diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index b8d5d2e6ee..59e63e9eae 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -259,15 +259,26 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) { static int32_t ignoreTime = 0; if (++ignoreTime > 10 && 0 == taosRand() % 9) { + if (ctx->msgType == TDMT_SCH_FETCH) { + qwBuildAndSendErrorRsp(TDMT_SCH_LINK_BROKEN, &ctx->ctrlConnInfo, TSDB_CODE_RPC_BROKEN_LINK); + qwBuildAndSendErrorRsp(ctx->msgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); + *rsped = true; + + taosSsleep(3); + return; + } + +#if 0 SRpcHandleInfo *pConn = ((ctx->msgType == TDMT_SCH_FETCH || ctx->msgType == TDMT_SCH_MERGE_FETCH) ? &ctx->dataConnInfo - : &ctx->ctrlConnInfo); + : &ctx->ctrlConnInfo); qwBuildAndSendErrorRsp(ctx->msgType + 1, pConn, TSDB_CODE_RPC_BROKEN_LINK); - + qwBuildAndSendDropMsg(QW_FPARAMS(), pConn); *rsped = true; - + return; +#endif } } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index b7f9272bdc..d002b5dfa9 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -299,6 +299,7 @@ typedef struct SSchJob { SExecResult execRes; void *fetchRes; // TODO free it or not bool fetched; + bool noMoreRetry; int64_t resNumOfRows; // from int32_t to int64_t SSchResInfo userRes; char *sql; @@ -333,8 +334,8 @@ extern SSchedulerMgmt schMgmt; ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \ (!SCH_IS_DATA_BIND_QRY_TASK(_task))) -#define SCH_UPDATE_REDICT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) -#define SCH_GET_REDICT_CODE(job, _code) (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode) +#define SCH_UPDATE_REDIRECT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) +#define SCH_GET_REDIRECT_CODE(job, _code) (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode) #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) @@ -398,7 +399,7 @@ extern SSchedulerMgmt schMgmt; (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_LOW_LEVEL_NETWORK_ERR((_job), (_task), (_code)) || SCH_TASK_RETRY_NETWORK_ERR((_task), (_code)))) #define SCH_TASK_NEED_RETRY(_msgType, _code) \ ((SCH_REDIRECT_MSGTYPE(_msgType) && SCH_NETWORK_ERR(_code)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) - + #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum) #define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) @@ -522,6 +523,11 @@ extern SSchedulerMgmt schMgmt; } \ } while (0) +#define SCH_RESET_JOB_LEVEL_IDX(_job) do { \ + (_job)->levelIdx = (_job)->levelNum - 1; \ + SCH_JOB_DLOG("set job levelIdx to %d", (_job)->levelIdx); \ +} while (0) + void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); void schCleanClusterHb(void *pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); @@ -603,6 +609,10 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync); int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode); int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp); +int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode); +int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode); +void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask); +int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, int32_t rspCode); extern SSchDebug gSCHDebug; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index b2c90dc67f..a36cd6747c 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -296,7 +296,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { } pJob->levelNum = levelNum; - pJob->levelIdx = levelNum - 1; + SCH_RESET_JOB_LEVEL_IDX(pJob); SSchLevel level = {0}; SNodeListNode *plans = NULL; @@ -828,8 +828,9 @@ int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) { SCH_LOCK(SCH_WRITE, &pJob->resLock); if (pJob->fetched) { SCH_UNLOCK(SCH_WRITE, &pJob->resLock); - SCH_TASK_ELOG("already fetched while got error %s", tstrerror(rspCode)); - SCH_ERR_JRET(rspCode); + pJob->noMoreRetry = true; + SCH_JOB_ELOG("already fetched while got error %s", tstrerror(rspCode)); + SCH_ERR_RET(rspCode); } SCH_UNLOCK(SCH_WRITE, &pJob->resLock); @@ -839,12 +840,56 @@ int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) { return TSDB_CODE_SUCCESS; } +int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode) { + SCH_ERR_RET(schChkResetJobRetry(pJob, rspCode)); + + int32_t numOfLevels = taosArrayGetSize(pJob->levels); + for (int32_t i = 0; i < numOfLevels; ++i) { + SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + + pLevel->taskExecDoneNum = 0; + pLevel->taskLaunchedNum = 0; + + int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); + for (int32_t j = 0; j < numOfTasks; ++j) { + SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); + SCH_LOCK_TASK(pTask); + SCH_ERR_RET(schChkUpdateRedirectCtx(pJob, pTask, NULL, rspCode)); + qClearSubplanExecutionNode(pTask->plan); + schResetTaskForRetry(pJob, pTask); + SCH_UNLOCK_TASK(pTask); + } + } + + SCH_RESET_JOB_LEVEL_IDX(pJob); + + return TSDB_CODE_SUCCESS; +} + int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - SCH_ERR_JRET(schChkResetJobRetry(pJob, rspCode)); + taosMemoryFreeClear(pMsg->pData); + taosMemoryFreeClear(pMsg->pEpSet); + SCH_UNLOCK_TASK(pTask); + + SCH_TASK_DLOG("start to redirect all job tasks cause of error: %s", tstrerror(rspCode)); + + SCH_ERR_JRET(schResetJobForRetry(pJob, rspCode)); + + SCH_ERR_JRET(schLaunchJob(pJob)); + + SCH_LOCK_TASK(pTask); + + SCH_RET(code); + +_return: + + SCH_LOCK_TASK(pTask); + + SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index af206aa46e..80fdc7594c 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -416,6 +416,7 @@ _return: int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; int32_t msgType = pMsg->msgType; + char *msg = pMsg->pData; bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode)); if (SCH_IS_QUERY_JOB(pJob)) { @@ -426,7 +427,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1); if (SCH_JOB_NEED_RETRY(pJob, pTask, reqType, rspCode)) { - SCH_RET(schHandleJobRetry()); + SCH_RET(schHandleJobRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode)); } else if (SCH_TASKSET_NEED_RETRY(pJob, pTask, reqType, rspCode)) { SCH_RET(schHandleTaskSetRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode)); } @@ -434,6 +435,12 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa pTask->redirectCtx.inRedirect = false; SCH_RET(schProcessResponseMsg(pJob, pTask, execId, pMsg, rspCode)); + +_return: + + taosMemoryFreeClear(msg); + + SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index a3194c7ff7..207753ae25 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -378,7 +378,8 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, if (lastTime > tsMaxRetryWaitTime) { SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); - SCH_ERR_RET(SCH_GET_REDICT_CODE(pJob, rspCode)); + pJob->noMoreRetry = true; + SCH_ERR_RET(SCH_GET_REDIRECT_CODE(pJob, rspCode)); } pCtx->periodMs *= tsRedirectFactor; @@ -408,16 +409,16 @@ void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { pTask->waitRetry = true; schDropTaskOnExecNode(pJob, pTask); + if (pTask->delayTimer) { + taosTmrStopA(&pTask->delayTimer); + } taosHashClear(pTask->execNodes); schRemoveTaskFromExecList(pJob, pTask); schDeregisterTaskHb(pJob, pTask); - atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); - if (SCH_TASK_EXEC_DONE(pTask)) { - atomic_sub_fetch_32(&pTask->level->taskExecDoneNum, 1); - } taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; pTask->lastMsgType = 0; + pTask->childReady = 0; memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); } @@ -427,7 +428,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_TASK_DLOG("task will be redirected now, status:%s, code:%s", SCH_GET_TASK_STATUS_STR(pTask), tstrerror(rspCode)); if (!NO_RET_REDIRECT_ERROR(rspCode)) { - SCH_UPDATE_REDICT_CODE(pJob, rspCode); + SCH_UPDATE_REDIRECT_CODE(pJob, rspCode); } SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL, rspCode)); @@ -496,6 +497,17 @@ int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, i } } + SCH_TASK_DLOG("start to redirect current task set cause of error: %s", tstrerror(rspCode)); + + for (int32_t i = 0; i < pJob->levelNum; ++i) { + SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + + pLevel->taskExecDoneNum = 0; + pLevel->taskLaunchedNum = 0; + } + + SCH_RESET_JOB_LEVEL_IDX(pJob); + code = schDoTaskRedirect(pJob, pTask, pData, rspCode); taosMemoryFreeClear(pData->pData); @@ -609,6 +621,13 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { */ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { + if (pJob->noMoreRetry) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry since job no more retry, retryTimes:%d/%d", pTask->retryTimes, + pTask->maxRetryTimes); + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SCH_TIMEOUT_ERROR == errCode) { pTask->maxExecTimes++; pTask->maxRetryTimes++; From 066a173d28e01ad8a10db36b8bf2da673c1c3440 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 18:39:45 +0800 Subject: [PATCH 112/176] fix(tmq): fix the invalid write. --- source/client/src/clientTmq.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index a627d5f190..ade16b7f1a 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -113,6 +113,7 @@ struct tmq_t { typedef struct SAskEpInfo { int32_t code; + tsem_t sem; } SAskEpInfo; enum { @@ -2138,8 +2139,9 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { } tsem_wait(&pInfo->sem); - code = pInfo->code; + + tsem_destroy(&pInfo->sem); taosMemoryFree(pInfo); tscDebug("consumer:0x%"PRIx64" sync commit done, code:%s", tmq->consumerId, tstrerror(code)); @@ -2159,7 +2161,7 @@ void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* par tDeleteSMqAskEpRsp(&rsp); } - tsem_post(&pTmq->rspSem); + tsem_post(&pInfo->sem); } void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) { @@ -2186,11 +2188,13 @@ void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* p int32_t doAskEp(tmq_t* pTmq) { SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); + tsem_init(&pInfo->sem, 0, 0); asyncAskEp(pTmq, updateEpCallbackFn, pInfo); - tsem_wait(&pTmq->rspSem); + tsem_wait(&pInfo->sem); int32_t code = pInfo->code; + tsem_destroy(&pInfo->sem); taosMemoryFree(pInfo); return code; } From afcc3d2064c0374bed64af24ee78ffbdd4b1f9d0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 31 Mar 2023 19:02:38 +0800 Subject: [PATCH 113/176] opti:code for tmq --- source/dnode/vnode/src/tq/tqScan.c | 5 +---- tests/system-test/7-tmq/subscribeStb3.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index acacec27a9..e4acf49635 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -88,11 +88,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs return -1; } - tqDebug("consumer:0x%"PRIx64" vgId:%d tmq task executed, rows:%"PRId64", total blocks:%d", pHandle->consumerId, vgId, - pDataBlock->info.rows, pRsp->blockNum); - + tqDebug("consumer:0x%"PRIx64" vgId:%d tmq task executed, total blocks:%d, pDataBlock:%p", pHandle->consumerId, vgId, pRsp->blockNum, pDataBlock); // current scan should be stopped asap, since the rebalance occurs. - if (pDataBlock == NULL) { break; } diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 33a7c506c5..ef7ca8ff5d 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -215,7 +215,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -328,7 +329,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -415,7 +417,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -502,7 +505,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' From ae4234cce07ab985ad5a79d57b1610fc95e7ab66 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 19:36:27 +0800 Subject: [PATCH 114/176] fix(tmq): add lock when check table list. --- source/libs/executor/src/scanoperator.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 667a6d6896..2907c8d056 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -771,7 +771,8 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { // scan table one by one sequentially if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { - int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); + int32_t numOfTables = 0;//tableListGetSize(pTaskInfo->pTableInfoList); + STableKeyInfo tInfo = {0}; while (1) { SSDataBlock* result = doGroupedTableScan(pOperator); @@ -781,14 +782,21 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { // if no data, switch to next table and continue scan pInfo->currentTable++; + + taosRLockLatch(&pTaskInfo->lock); + numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); + if (pInfo->currentTable >= numOfTables) { qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); + taosRUnLockLatch(&pTaskInfo->lock); return NULL; } - STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable); - tsdbSetTableList(pInfo->base.dataReader, pTableInfo, 1); - qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", pTableInfo->uid, numOfTables, + tInfo = *(STableKeyInfo*) tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable); + taosRUnLockLatch(&pTaskInfo->lock); + + tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1); + qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); From f1384206f387192c6ca9c9f6415414abcac9c7b9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Mar 2023 19:46:10 +0800 Subject: [PATCH 115/176] refactor: do some internal refactor. --- source/dnode/mnode/impl/src/mndTrans.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 65ba76b21f..9d8a662b6b 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1086,12 +1086,12 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { goto _OVER; } - pTrans->lastErrorNo = pRsp->code; - STransAction *pAction = taosArrayGet(pArray, action); if (pAction != NULL) { pAction->msgReceived = 1; pAction->errCode = pRsp->code; + pTrans->lastErrorNo = pRsp->code; + mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode); } else { From 02dc919b9d044c87084d6a81b406bbcfee3e260c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 31 Mar 2023 20:47:10 +0800 Subject: [PATCH 116/176] fix: job status issue --- source/libs/scheduler/src/schJob.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index a36cd6747c..e7bfe95795 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -83,6 +83,10 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { oriStatus = SCH_GET_JOB_STATUS(pJob); if (oriStatus == newStatus) { + if (JOB_TASK_STATUS_FETCH == newStatus) { + return code; + } + SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -116,7 +120,8 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_FETCH: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && - newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) { + newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC && + newStatus != JOB_TASK_STATUS_FETCH) { SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } @@ -988,7 +993,7 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq SCH_ERR_RET(TSDB_CODE_APP_ERROR); } - if (status != JOB_TASK_STATUS_PART_SUCC) { + if (status != JOB_TASK_STATUS_PART_SUCC && status != JOB_TASK_STATUS_FETCH) { SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } From 23e8edd1481ba548ff61a860c47dc34ee42d3687 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 1 Apr 2023 00:14:01 +0800 Subject: [PATCH 117/176] fix(tmq): set the default commit callback fn. --- source/client/src/clientTmq.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ade16b7f1a..59a407656d 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -198,7 +198,7 @@ typedef struct { int32_t waitingRspNum; int32_t totalRspNum; int32_t code; - tmq_commit_cb* userCb; + tmq_commit_cb* callbackFn; /*SArray* successfulOffsets;*/ /*SArray* failedOffsets;*/ void* userParam; @@ -590,7 +590,7 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* p pParamSet->refId = tmq->refId; pParamSet->epoch = tmq->epoch; - pParamSet->userCb = pCommitFp; + pParamSet->callbackFn = pCommitFp; pParamSet->userParam = userParam; int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); @@ -656,7 +656,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us pParamSet->refId = tmq->refId; pParamSet->epoch = tmq->epoch; - pParamSet->userCb = pCommitFp; + pParamSet->callbackFn = pCommitFp; pParamSet->userParam = userParam; // init as 1 to prevent concurrency issue @@ -810,6 +810,12 @@ OVER: taosReleaseRef(tmqMgmt.rsetId, refId); } +static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { + if (code != 0) { + tscDebug("consumer:0x%"PRIx64", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); + } +} + int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { STaosQall* qall = taosAllocateQall(); taosReadAllQitems(pTmq->delayedTask, qall); @@ -833,7 +839,9 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); taosTmrReset(tmqAssignAskEpTask, 1000, pRefId, tmqMgmt.timer, &pTmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - asyncCommitAllOffsets(pTmq, pTmq->commitCb, pTmq->commitCbUserParam); + tmq_commit_cb* pCallbackFn = pTmq->commitCb? pTmq->commitCb:defaultCommitCbFn; + + asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); *pRefId = pTmq->refId; @@ -1029,8 +1037,6 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->status = TMQ_CONSUMER_STATUS__INIT; pTmq->pollCnt = 0; pTmq->epoch = 0; - /*pTmq->epStatus = 0;*/ - /*pTmq->epSkipCnt = 0;*/ // set conf strcpy(pTmq->clientId, conf->clientId); @@ -2279,7 +2285,7 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { } // if no more waiting rsp - pParamSet->userCb(tmq, pParamSet->code, pParamSet->userParam); + pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); taosMemoryFree(pParamSet); taosReleaseRef(tmqMgmt.rsetId, refId); From addabf3ba7f565ec9a348434215c60bc10ea0ed9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 2 Apr 2023 15:25:31 +0800 Subject: [PATCH 118/176] refactor: push down the tableListinfo to scanner. --- source/common/src/tdatablock.c | 44 ++------- source/libs/executor/inc/executorimpl.h | 25 +++--- source/libs/executor/src/cachescanoperator.c | 15 ++-- source/libs/executor/src/executor.c | 4 +- source/libs/executor/src/executorimpl.c | 25 +++--- source/libs/executor/src/scanoperator.c | 95 ++++++++++---------- source/libs/executor/src/sysscanoperator.c | 13 +-- 7 files changed, 98 insertions(+), 123 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index cac8f73042..a65f30f023 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1035,6 +1035,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return TSDB_CODE_SUCCESS; } +#if 0 typedef struct SHelper { int32_t index; union { @@ -1083,59 +1084,20 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param; - // SSDataBlock* pDataBlock = pHelper->pDataBlock; - SHelper* left = (SHelper*)p1; SHelper* right = (SHelper*)p2; SArray* pInfo = pHelper->orderInfo; int32_t offset = 0; - // for(int32_t i = 0; i < pInfo->size; ++i) { - // SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0); - // SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); - - // if (pColInfoData->hasNull) { - // bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); - // bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); - // if (leftNull && rightNull) { - // continue; // continue to next slot - // } - // - // if (rightNull) { - // return pHelper->nullFirst? 1:-1; - // } - // - // if (leftNull) { - // return pHelper->nullFirst? -1:1; - // } - // } - - // void* left1 = colDataGetData(pColInfoData, left); - // void* right1 = colDataGetData(pColInfoData, right); - - // switch(pColInfoData->info.type) { - // case TSDB_DATA_TYPE_INT: { int32_t leftx = *(int32_t*)left->pData; //*(int32_t*)(left->pData + offset); int32_t rightx = *(int32_t*)right->pData; //*(int32_t*)(right->pData + offset); - // offset += pColInfoData->info.bytes; if (leftx == rightx) { - // break; return 0; } else { - // if (pOrder->order == TSDB_ORDER_ASC) { return (leftx < rightx) ? -1 : 1; - // } else { - // return (leftx < rightx)? 1:-1; - // } } - // } - // default: - // assert(0); - // } - // } - return 0; } @@ -1179,6 +1141,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF // destroyTupleIndex(index); return 0; } +#endif void blockDataCleanup(SSDataBlock* pDataBlock) { blockDataEmpty(pDataBlock); @@ -1887,6 +1850,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } +#if 0 void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag) { SArray* dataBlocks = taosArrayInit(1, sizeof(SSDataBlock*)); taosArrayPush(dataBlocks, &pBlock); @@ -1979,6 +1943,8 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { } } +#endif + // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) { int32_t size = 2048; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 4eceefcd51..cea7216dd5 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -175,7 +175,7 @@ struct SExecTaskInfo { int64_t version; // used for stream to record wal version, why not move to sschemainfo SStreamTaskInfo streamInfo; SSchemaInfo schemaInfo; - STableListInfo* pTableInfoList; // this is a table list +// STableListInfo* pTableInfoList; // this is a table list const char* sql; // query sql string jmp_buf env; // jump to this position when error happens. EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] @@ -323,6 +323,8 @@ typedef struct STableScanBase { int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; SLimitInfo limitInfo; + // there are more than one table list exists in one task, if only one vnode exists. + STableListInfo* pTableInfoList; } STableScanBase; typedef struct STableScanInfo { @@ -363,11 +365,12 @@ typedef struct STableMergeScanInfo { } STableMergeScanInfo; typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; - SColMatchInfo matchInfo; - int32_t curPos; - SReadHandle readHandle; + SColumnInfo* pCols; + SSDataBlock* pRes; + SColMatchInfo matchInfo; + int32_t curPos; + SReadHandle readHandle; + STableListInfo* pTableInfoList; } STagScanInfo; typedef enum EStreamScanMode { @@ -753,9 +756,9 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // clang-format off SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableList, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); @@ -773,7 +776,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); @@ -787,9 +790,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index d42b348fd8..226b3dacd0 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -37,6 +37,7 @@ typedef struct SCacheRowsScanInfo { SSDataBlock* pBufferredRes; SArray* pUidList; int32_t indexOfBufferedRes; + STableListInfo* pTableList; } SCacheRowsScanInfo; static SSDataBlock* doScanCache(SOperatorInfo* pOperator); @@ -47,7 +48,7 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM #define SCAN_ROW_TYPE(_t) ((_t)? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -75,20 +76,18 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe goto _error; } - STableListInfo* pTableList = pTaskInfo->pTableInfoList; - - int32_t totalTables = tableListGetSize(pTableList); + int32_t totalTables = tableListGetSize(pTableListInfo); int32_t capacity = 0; pInfo->pUidList = taosArrayInit(4, sizeof(int64_t)); // partition by tbname - if (oneTableForEachGroup(pTableList) || (totalTables == 1)) { + if (oneTableForEachGroup(pTableListInfo) || (totalTables == 1)) { pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL | SCAN_ROW_TYPE(pScanNode->ignoreNull); - STableKeyInfo* pList = tableListGetInfo(pTableList, 0); + STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); - uint64_t suid = tableListGetSuid(pTableList); + uint64_t suid = tableListGetSuid(pTableListInfo); code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { @@ -136,7 +135,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { SCacheRowsScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableListInfo* pTableList = pTaskInfo->pTableInfoList; + STableListInfo* pTableList = pInfo->pTableList; uint64_t suid = tableListGetSuid(pTableList); int32_t size = tableListGetSize(pTableList); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ff11dadf22..c8fce05341 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -404,7 +404,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } } - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + STableListInfo* pTableListInfo = ((STableScanInfo*)pScanInfo->pTableScanOp->info)->base.pTableInfoList; taosWLockLatch(&pTaskInfo->lock); for (int32_t i = 0; i < numOfQualifiedTables; ++i) { @@ -1083,7 +1083,7 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + STableListInfo* pTableListInfo = NULL;//pTaskInfo->pTableInfoList; const char* id = GET_TASKID(pTaskInfo); pTaskInfo->streamInfo.prepareStatus = *pOffset; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8068590dad..6e14643e60 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1986,7 +1986,6 @@ SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t v pTaskInfo->schemaInfo.dbname = taosStrdup(dbFName); pTaskInfo->execModel = model; - pTaskInfo->pTableInfoList = tableListCreate(); pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); @@ -2101,7 +2100,6 @@ bool groupbyTbname(SNodeList* pGroupList) { SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SNode* pTagCond, SNode* pTagIndexCond, const char* pUser) { int32_t type = nodeType(pPhyNode); - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; const char* idstr = GET_TASKID(pTaskInfo); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -2115,6 +2113,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTableScanNode->groupSort = true; } + STableListInfo* pTableListInfo = tableListCreate(); int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); @@ -2130,7 +2129,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); + pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); if (NULL == pOperator) { pTaskInfo->code = terrno; return NULL; @@ -2140,6 +2139,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); @@ -2155,7 +2155,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); + pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); if (NULL == pOperator) { pTaskInfo->code = terrno; return NULL; @@ -2168,6 +2168,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); + if (pHandle->vnode) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, @@ -2190,7 +2192,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); - pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo); + pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); @@ -2199,7 +2201,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - + STableListInfo* pTableListInfo = tableListCreate(); int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2211,6 +2213,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); if (pBlockNode->tableType == TSDB_SUPER_TABLE) { SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); @@ -2231,9 +2234,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo tableListAddTableInfo(pTableListInfo, pBlockNode->uid, 0); } - pOperator = createDataBlockInfoScanOperator(pHandle, pBlockNode, pTaskInfo); + pOperator = createDataBlockInfoScanOperator(pHandle, pBlockNode, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); @@ -2248,7 +2252,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - pOperator = createCacherowsScanOperator(pScanNode, pHandle, pTaskInfo); + pOperator = createCacherowsScanOperator(pScanNode, pHandle, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { pOperator = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo); } else { @@ -2416,6 +2420,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT if (NULL == pDeleterParam) { return TSDB_CODE_OUT_OF_MEMORY; } +#if 0 int32_t tbNum = tableListGetSize(pTask->pTableInfoList); pDeleterParam->suid = tableListGetSuid(pTask->pTableInfoList); @@ -2432,6 +2437,8 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT } *pParam = pDeleterParam; +#endif + break; } default: @@ -2481,8 +2488,6 @@ static void freeBlock(void* pParam) { void doDestroyTask(SExecTaskInfo* pTaskInfo) { qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); - - pTaskInfo->pTableInfoList = tableListDestroy(pTaskInfo->pTableInfoList); destroyOperatorInfo(pTaskInfo->pRoot); cleanupTableSchemaInfo(&pTaskInfo->schemaInfo); cleanupStreamInfo(&pTaskInfo->streamInfo); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2907c8d056..1fffe14b02 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -676,7 +676,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (pBlock->info.id.uid) { - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + pBlock->info.id.groupId = 0;//getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); } uint32_t status = 0; @@ -784,7 +784,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { pInfo->currentTable++; taosRLockLatch(&pTaskInfo->lock); - numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); + numOfTables = tableListGetSize(pInfo->base.pTableInfoList); if (pInfo->currentTable >= numOfTables) { qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); @@ -792,7 +792,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return NULL; } - tInfo = *(STableKeyInfo*) tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable); + tInfo = *(STableKeyInfo*) tableListGetInfo(pInfo->base.pTableInfoList, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1); @@ -804,14 +804,14 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } } else { // scan table group by group sequentially if (pInfo->currentGroupId == -1) { - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableInfoList)) { setOperatorCompleted(pOperator); return NULL; } int32_t num = 0; STableKeyInfo* pList = NULL; - tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); + tableListGetGroupList(pInfo->base.pTableInfoList, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, @@ -830,7 +830,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return result; } - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableInfoList)) { setOperatorCompleted(pOperator); return NULL; } @@ -841,7 +841,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); + tableListGetGroupList(pInfo->base.pTableInfoList, pInfo->currentGroupId, &pList, &num); tsdbSetTableList(pInfo->base.dataReader, pList, num); tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); @@ -866,25 +866,30 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr return 0; } +static void destroyTableScanBase(STableScanBase* pBase) { + cleanupQueryTableDataCond(&pBase->cond); + + tsdbReaderClose(pBase->dataReader); + pBase->dataReader = NULL; + + if (pBase->matchInfo.pList != NULL) { + taosArrayDestroy(pBase->matchInfo.pList); + } + + tableListDestroy(pBase->pTableInfoList); + taosLRUCacheCleanup(pBase->metaCache.pTableMetaEntryCache); + cleanupExprSupp(&pBase->pseudoSup); +} + static void destroyTableScanOperatorInfo(void* param) { STableScanInfo* pTableScanInfo = (STableScanInfo*)param; blockDataDestroy(pTableScanInfo->pResBlock); - cleanupQueryTableDataCond(&pTableScanInfo->base.cond); - - tsdbReaderClose(pTableScanInfo->base.dataReader); - pTableScanInfo->base.dataReader = NULL; - - if (pTableScanInfo->base.matchInfo.pList != NULL) { - taosArrayDestroy(pTableScanInfo->base.matchInfo.pList); - } - - taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache); - cleanupExprSupp(&pTableScanInfo->base.pseudoSup); + destroyTableScanBase(&pTableScanInfo->base); taosMemoryFreeClear(param); } SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -940,6 +945,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; + pInfo->base.pTableInfoList = pTableListInfo; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) { code = terrno; @@ -1059,7 +1065,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU if (hasNext) { /*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableInfoList, pBlock->info.id.uid); } tsdbReaderClose(pReader); @@ -1080,7 +1086,8 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, } static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) { - return getTableGroupId(pInfo->pTableScanOp->pTaskInfo->pTableInfoList, uid); + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + return getTableGroupId(pTableScanInfo->base.pTableInfoList, uid); } static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { @@ -1554,7 +1561,8 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.version = pBlock->info.version; - pInfo->pRes->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + pInfo->pRes->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableInfoList, pBlock->info.id.uid); // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { @@ -2314,6 +2322,7 @@ _end: static void destroyStreamScanOperatorInfo(void* param) { SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param; + if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) { destroyOperatorInfo(pStreamScan->pTableScanOp); } @@ -2343,7 +2352,7 @@ static void destroyStreamScanOperatorInfo(void* param) { } SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { SArray* pColIds = NULL; SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -2411,7 +2420,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } if (pHandle->vnode) { - SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); + SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info; if (pHandle->version > 0) { pTSInfo->base.cond.endVersion = pHandle->version; @@ -2419,7 +2428,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys STableKeyInfo* pList = NULL; int32_t num = 0; - tableListGetGroupList(pTaskInfo->pTableInfoList, 0, &pList, &num); + tableListGetGroupList(pTableListInfo, 0, &pList, &num); if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; @@ -2450,7 +2459,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys // set the extract column id to streamHandle tqReaderSetColIdList(pInfo->tqReader, pColIds); - SArray* tableIdList = extractTableIdList(pTaskInfo->pTableInfoList); + SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableInfoList); code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); @@ -2525,7 +2534,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->pRes; blockDataCleanup(pRes); - int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); + int32_t size = tableListGetSize(pInfo->pTableInfoList); if (size == 0) { setTaskStatus(pTaskInfo, TASK_COMPLETED); return NULL; @@ -2537,7 +2546,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { metaReaderInit(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - STableKeyInfo* item = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->curPos); + STableKeyInfo* item = tableListGetInfo(pInfo->pTableInfoList, pInfo->curPos); int32_t code = metaGetTableEntryByUid(&mr, item->uid); tDecoderClear(&mr.coder); if (code != TSDB_CODE_SUCCESS) { @@ -2657,7 +2666,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx); int64_t st = taosGetTimestampUs(); - void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); + void* p = tableListGetInfo(pInfo->base.pTableInfoList, readIdx + pInfo->tableStartIndex); SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { @@ -2714,7 +2723,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { continue; } - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + pBlock->info.id.groupId = getTableGroupId(pInfo->base.pTableInfoList, pBlock->info.id.uid); pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; @@ -2770,10 +2779,10 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; { - size_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); + size_t numOfTables = tableListGetSize(pInfo->base.pTableInfoList); int32_t i = pInfo->tableStartIndex + 1; for (; i < numOfTables; ++i) { - STableKeyInfo* tableKeyInfo = tableListGetInfo(pTaskInfo->pTableInfoList, i); + STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableInfoList, i); if (tableKeyInfo->groupId != pInfo->groupId) { break; } @@ -2907,7 +2916,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - size_t tableListSize = tableListGetSize(pTaskInfo->pTableInfoList); + size_t tableListSize = tableListGetSize(pInfo->base.pTableInfoList); if (!pInfo->hasGroupId) { pInfo->hasGroupId = true; @@ -2916,7 +2925,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { return NULL; } pInfo->tableStartIndex = 0; - pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex))->groupId; + pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pInfo->base.pTableInfoList, pInfo->tableStartIndex))->groupId; startGroupTableMergeScan(pOperator); } @@ -2941,7 +2950,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex)->groupId; + pInfo->groupId = tableListGetInfo(pInfo->base.pTableInfoList, pInfo->tableStartIndex)->groupId; startGroupTableMergeScan(pOperator); resetLimitInfoForNextGroup(&pInfo->limitInfo); } @@ -2963,9 +2972,6 @@ void destroyTableMergeScanOperatorInfo(void* param) { p->dataReader = NULL; } - tsdbReaderClose(pTableScanInfo->base.dataReader); - pTableScanInfo->base.dataReader = NULL; - taosArrayDestroy(pTableScanInfo->sortSourceParams); tsortDestroySortHandle(pTableScanInfo->pSortHandle); pTableScanInfo->pSortHandle = NULL; @@ -2974,20 +2980,14 @@ void destroyTableMergeScanOperatorInfo(void* param) { SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i); taosMemoryFree(pCond->colList); } - taosArrayDestroy(pTableScanInfo->queryConds); - if (pTableScanInfo->base.matchInfo.pList != NULL) { - taosArrayDestroy(pTableScanInfo->base.matchInfo.pList); - } + taosArrayDestroy(pTableScanInfo->queryConds); + destroyTableScanBase(&pTableScanInfo->base); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); taosArrayDestroy(pTableScanInfo->pSortInfo); - cleanupExprSupp(&pTableScanInfo->base.pseudoSup); - - taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache); - taosMemoryFreeClear(param); } @@ -3006,7 +3006,7 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla } SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -3048,6 +3048,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->base.limitInfo.limit.limit = -1; pInfo->base.limitInfo.slimit.limit = -1; + pInfo->base.pTableInfoList = pTableListInfo; pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 68e5cf6ef8..161811531a 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -83,10 +83,11 @@ typedef struct MergeIndex { } MergeIndex; typedef struct SBlockDistInfo { - SSDataBlock* pResBlock; - STsdbReader* pHandle; - SReadHandle readHandle; - uint64_t uid; // table uid + SSDataBlock* pResBlock; + STsdbReader* pHandle; + SReadHandle readHandle; + STableListInfo* pTableListInfo; + uint64_t uid; // table uid } SBlockDistInfo; static int32_t sysChkFilter__Comm(SNode* pNode); @@ -2245,7 +2246,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC } SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2263,7 +2264,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi goto _error; } - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + pInfo->pTableListInfo = pTableListInfo; size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); From a887ccda897ac35f4884ff80170a9caa129c9067 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 2 Apr 2023 15:30:56 +0800 Subject: [PATCH 119/176] fix(query): destroy the tablelist in cache scanner. --- source/libs/executor/src/cachescanoperator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 226b3dacd0..0588b9c53a 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -280,6 +280,7 @@ void destroyCacheScanOperator(void* param) { taosMemoryFree(pInfo->pSlotIds); taosArrayDestroy(pInfo->pUidList); taosArrayDestroy(pInfo->matchInfo.pList); + tableListDestroy(pInfo->pTableList); if (pInfo->pLastrowReader != NULL) { pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); From 167a56f8d04421717d45c0dc8abaa6ce8867be7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 2 Apr 2023 21:25:54 +0800 Subject: [PATCH 120/176] fix(query): set the correct group table list. --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/scanoperator.c | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index cea7216dd5..94a87a018e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -760,7 +760,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6e14643e60..a094c146de 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2210,7 +2210,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTaskInfo); + pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; STableListInfo* pTableListInfo = tableListCreate(); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1fffe14b02..4ce9ed0726 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -676,7 +676,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (pBlock->info.id.uid) { - pBlock->info.id.groupId = 0;//getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableInfoList, pBlock->info.id.uid); } uint32_t status = 0; @@ -2607,10 +2607,11 @@ static void destroyTagScanOperatorInfo(void* param) { STagScanInfo* pInfo = (STagScanInfo*)param; pInfo->pRes = blockDataDestroy(pInfo->pRes); taosArrayDestroy(pInfo->matchInfo.pList); + pInfo->pTableInfoList = tableListDestroy(pInfo->pTableInfoList); taosMemoryFreeClear(param); } -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -2633,6 +2634,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } + pInfo->pTableInfoList = pTableListInfo; pInfo->pRes = createDataBlockFromDescNode(pDescNode); pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; From b029c8f15787d686d507a30d661d45bdb236acf5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 2 Apr 2023 22:34:51 +0800 Subject: [PATCH 121/176] fix:remove useless setmsg --- source/dnode/vnode/inc/vnode.h | 2 -- source/dnode/vnode/src/tq/tqRead.c | 24 +++++++----------------- source/libs/executor/src/scanoperator.c | 1 - 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e4af676259..a9e5fe628b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -232,10 +232,8 @@ typedef struct SSnapContext { } SSnapContext; typedef struct STqReader { -// int64_t ver; SPackedData msg2; - int8_t setMsg; SSubmitReq2 submit; int32_t nextBlk; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6d4a8a1c76..4f92330000 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -329,31 +329,24 @@ void tqNextBlock(STqReader* pReader, SFetchRet* ret) { } int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { -// ASSERT(pReader->msg2.msgStr == NULL && msgStr && msgLen && (ver >= 0)); - pReader->msg2.msgStr = msgStr; pReader->msg2.msgLen = msgLen; pReader->msg2.ver = ver; -// pReader->ver = ver; tqDebug("tq reader set msg %p %d", msgStr, msgLen); - - if (pReader->setMsg == 0) { - SDecoder decoder; - tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); - if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { - tDecoderClear(&decoder); - tqError("DecodeSSubmitReq2 error, msgLen:%d, ver:%"PRId64, msgLen, ver); - return -1; - } + SDecoder decoder; + tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); + if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { tDecoderClear(&decoder); - pReader->setMsg = 1; + tqError("DecodeSSubmitReq2 error, msgLen:%d, ver:%"PRId64, msgLen, ver); + return -1; } + tDecoderClear(&decoder); return 0; } bool tqNextDataBlock2(STqReader* pReader) { - if (pReader->msg2.msgStr == NULL || pReader->setMsg != 1) { + if (pReader->msg2.msgStr == NULL) { return false; } @@ -372,7 +365,6 @@ bool tqNextDataBlock2(STqReader* pReader) { } tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->setMsg = 0; pReader->nextBlk = 0; pReader->msg2.msgStr = NULL; @@ -381,7 +373,6 @@ bool tqNextDataBlock2(STqReader* pReader) { bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { if (pReader->msg2.msgStr == NULL) return false; - ASSERT(pReader->setMsg == 1); int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { @@ -396,7 +387,6 @@ bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { } tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->setMsg = 0; pReader->nextBlk = 0; pReader->msg2.msgStr = NULL; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 734b5ad1fe..3b2d2188a6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2033,7 +2033,6 @@ FETCH_NEXT_BLOCK: int32_t current = pInfo->validBlockIndex++; SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); - /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, totBlockNum); From f977b27b88d6457b103f9af3ad5a922b97cb296c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 2 Apr 2023 22:38:18 +0800 Subject: [PATCH 122/176] fix:remove useless setmsg --- source/libs/executor/src/scanoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3b2d2188a6..d2b3a8c3c5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1635,7 +1635,6 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } pInfo->tqReader->msg2 = (SPackedData){0}; - pInfo->tqReader->setMsg = 0; pTaskInfo->streamInfo.submit = (SPackedData){0}; return NULL; } From 09df16f6e443c93fc4897a9a1a560b2cfadc6212 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 2 Apr 2023 22:47:26 +0800 Subject: [PATCH 123/176] fix(query): fix memory leak. --- source/libs/executor/src/exchangeoperator.c | 6 +++--- source/libs/executor/src/executorimpl.c | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 8caebd2a34..c855a104b2 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -70,7 +70,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, pTaskInfo->code); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } for (int32_t i = 0; i < totalSources; ++i) { @@ -576,7 +576,7 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, pTaskInfo->code); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } return TSDB_CODE_SUCCESS; @@ -629,7 +629,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, pTaskInfo->code); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a094c146de..e29432b808 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2119,6 +2119,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); qError("failed to createScanTableListInfo, code:%s, %s", tstrerror(code), idstr); return NULL; } @@ -2126,6 +2127,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo code = extractTableSchemaInfo(pHandle, &pTableScanNode->scan, pTaskInfo); if (code) { pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); return NULL; } From 858144145f5227ddc8b164028762b01b1175d90e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 2 Apr 2023 23:39:59 +0800 Subject: [PATCH 124/176] fix:memory leak --- source/common/src/tmsg.c | 1 + source/dnode/vnode/src/tq/tqRead.c | 1 + 2 files changed, 2 insertions(+) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7a238440a7..371073ed76 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7212,6 +7212,7 @@ void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) { tDestroySSubmitTbData(&aSubmitTbData[i], flag); } taosArrayDestroy(pReq->aSubmitTbData); + pReq->aSubmitTbData = NULL; } int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 4f92330000..54e4e393ec 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -290,6 +290,7 @@ void tqCloseReader(STqReader* pReader) { } // free hash taosHashCleanup(pReader->tbIdHash); + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); taosMemoryFree(pReader); } From 2d6436c57d714af026b86061d11dd33f5a227172 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 3 Apr 2023 11:24:22 +0800 Subject: [PATCH 125/176] fix:modify log & sleep 500ms if data is inserting while consume --- source/client/src/clientSml.c | 3 +-- source/client/src/clientTmq.c | 2 ++ source/dnode/vnode/src/tq/tq.c | 20 +++++++++++--------- source/dnode/vnode/src/tq/tqScan.c | 5 +++-- source/libs/executor/src/scanoperator.c | 2 +- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index a4ecc3c3df..40a685faf5 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -533,8 +533,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, - kv->key, colField[*index].type, kv->type); + uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key); return TSDB_CODE_TSC_INVALID_VALUE; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 59a407656d..e08be619ef 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1269,6 +1269,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; taosWriteQitem(tmq->mqueue, pRspWrapper); + }else if(code == TSDB_CODE_WAL_LOG_NOT_EXIST){ //poll data while insert + taosMsleep(500); } goto CREATE_MSG_FAIL; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 363aab8e45..f1935ad731 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -448,9 +448,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); if(code != 0) { - tDeleteSMqDataRsp(&dataRsp); - taosWUnLockLatch(&pTq->lock); - return TSDB_CODE_TMQ_CONSUMER_ERROR; + goto end; } // till now, all data has been transferred to consumer, new data needs to push client once arrived. @@ -461,16 +459,20 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, return code; } - taosWUnLockLatch(&pTq->lock); + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); // NOTE: this pHandle->consumerId may have been changed already. - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64 - ", ts:%" PRId64 ", reqId:0x%" PRIx64, - consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid, - dataRsp.rspOffset.ts, pRequest->reqId); - tDeleteSMqDataRsp(&dataRsp); +end: + { + char buf[80] = {0}; + tFormatOffset(buf, 80, &dataRsp.rspOffset); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d", + consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); + taosWUnLockLatch(&pTq->lock); + tDeleteSMqDataRsp(&dataRsp); + } return code; } diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index e4acf49635..dff162d527 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -82,13 +82,13 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; - tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq task start execute", pHandle->consumerId, vgId); + tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { tqError("consumer:0x%"PRIx64" vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); return -1; } - tqDebug("consumer:0x%"PRIx64" vgId:%d tmq task executed, total blocks:%d, pDataBlock:%p", pHandle->consumerId, vgId, pRsp->blockNum, pDataBlock); + tqDebug("consumer:0x%"PRIx64" vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId, pDataBlock); // current scan should be stopped asap, since the rebalance occurs. if (pDataBlock == NULL) { break; @@ -107,6 +107,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs } } + tqDebug("consumer:0x%"PRIx64" vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", pHandle->consumerId, vgId, pRsp->blockNum, totalRows); qStreamExtractOffset(task, &pRsp->rspOffset); return 0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 89ca7510e9..f723127711 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1672,7 +1672,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); //curVersion move to next, so currentOffset = curVersion - 1 if (ret.fetchType == FETCH_TYPE__DATA) { - qDebug("doQueueScan get data from log %"PRId64" rows, version:%" PRId64, pInfo->pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); + qDebug("doQueueScan get data from log %"PRId64" rows, version:%" PRId64, ret.data.info.rows, pTaskInfo->streamInfo.currentOffset.version); blockDataCleanup(pInfo->pRes); setBlockIntoRes(pInfo, &ret.data, true); if (pInfo->pRes->info.rows > 0) { From e2f58a555df706217999dce789ae36f6e3adf1c7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 3 Apr 2023 11:24:29 +0800 Subject: [PATCH 126/176] enh: convert the month and year in interval-offset to a database precision duration --- source/libs/parser/src/parTranslater.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f58a66af18..53816a5436 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -757,7 +757,8 @@ static bool isPrimaryKeyImpl(SNode* pExpr) { if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pFunc->funcType || FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType) { return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0)); - } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || FUNCTION_TYPE_IROWTS == pFunc->funcType) { + } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || + FUNCTION_TYPE_IROWTS == pFunc->funcType) { return true; } } @@ -3119,6 +3120,19 @@ static const char* getPrecisionStr(uint8_t precision) { return "unknown"; } +static void convertVarDuration(SValueNode* pOffset, uint8_t precision) { + const int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; + const int8_t units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND}; + + if (pOffset->unit == 'n') { + pOffset->datum.i = pOffset->datum.i * 31 * (NANOSECOND_PER_DAY / factors[precision]); + } else { + pOffset->datum.i = pOffset->datum.i * 365 * (NANOSECOND_PER_DAY / factors[precision]); + } + + pOffset->unit = units[precision]; +} + static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; @@ -3143,6 +3157,10 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG); } + + if (pOffset->unit == 'n' || pOffset->unit == 'y') { + convertVarDuration(pOffset, precision); + } } if (NULL != pInterval->pSliding) { From 91ae07b6c7768413f70071d844b88b8cc5acedd1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Apr 2023 11:57:17 +0800 Subject: [PATCH 127/176] fix(tmq): set the table list correctly. --- source/libs/executor/inc/executorimpl.h | 21 +- source/libs/executor/src/executor.c | 27 +- source/libs/executor/src/executorimpl.c | 593 ++---------------- source/libs/executor/src/scanoperator.c | 65 +- source/libs/executor/src/timewindowoperator.c | 7 - 5 files changed, 107 insertions(+), 606 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 94a87a018e..67c9db13ed 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -175,7 +175,6 @@ struct SExecTaskInfo { int64_t version; // used for stream to record wal version, why not move to sschemainfo SStreamTaskInfo streamInfo; SSchemaInfo schemaInfo; -// STableListInfo* pTableInfoList; // this is a table list const char* sql; // query sql string jmp_buf env; // jump to this position when error happens. EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] @@ -324,7 +323,7 @@ typedef struct STableScanBase { int32_t dataBlockLoadFlag; SLimitInfo limitInfo; // there are more than one table list exists in one task, if only one vnode exists. - STableListInfo* pTableInfoList; + STableListInfo* pTableListInfo; } STableScanBase; typedef struct STableScanInfo { @@ -370,7 +369,7 @@ typedef struct STagScanInfo { SColMatchInfo matchInfo; int32_t curPos; SReadHandle readHandle; - STableListInfo* pTableInfoList; + STableListInfo* pTableListInfo; } STagScanInfo; typedef enum EStreamScanMode { @@ -518,6 +517,16 @@ typedef struct SOptrBasicInfo { bool mergeResultBlock; } SOptrBasicInfo; +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + STableQueryInfo* current; + uint64_t groupId; + SGroupResInfo groupResInfo; + SExprSupp scalarExprSup; + bool groupKeyOptimized; +} SAggOperatorInfo; + typedef struct SIntervalAggOperatorInfo { SOptrBasicInfo binfo; // basic info SAggSupporter aggSup; // aggregate supporter @@ -734,6 +743,7 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int3 STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); +SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); @@ -837,9 +847,11 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); char* buildTaskId(uint64_t taskId, uint64_t queryId); +SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo); + int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t vgId, char* sql, EOPTR_EXEC_MODEL model); -int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* pTableListInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, @@ -853,7 +865,6 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); -bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index c8fce05341..b05d01f596 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -404,7 +404,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } } - STableListInfo* pTableListInfo = ((STableScanInfo*)pScanInfo->pTableScanOp->info)->base.pTableInfoList; + STableListInfo* pTableListInfo = ((STableScanInfo*)pScanInfo->pTableScanOp->info)->base.pTableListInfo; taosWLockLatch(&pTaskInfo->lock); for (int32_t i = 0; i < numOfQualifiedTables; ++i) { @@ -485,9 +485,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model) { - assert(pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - taosThreadOnce(&initPoolOnce, initRefPool); qDebug("start to create task, TID:0x%" PRIx64 " QID:0x%" PRIx64 ", vgId:%d", taskId, pSubplan->id.queryId, vgId); @@ -507,7 +505,12 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, if (handle) { void* pSinkParam = NULL; - code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle); + + SArray* pInfoList = getTableListInfo(*pTask); + STableListInfo* pTableListInfo = taosArrayGetP(pInfoList, 0); + taosArrayDestroy(pInfoList); + + code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTableListInfo, readHandle); if (code != TSDB_CODE_SUCCESS) { qError("failed to createDataSinkParam, vgId:%d, code:%s, %s", vgId, tstrerror(code), (*pTask)->id.str); goto _error; @@ -1083,7 +1086,6 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; - STableListInfo* pTableListInfo = NULL;//pTaskInfo->pTableInfoList; const char* id = GET_TASKID(pTaskInfo); pTaskInfo->streamInfo.prepareStatus = *pOffset; @@ -1095,19 +1097,12 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (subType == TOPIC_SUB_TYPE__COLUMN) { pOperator->status = OP_OPENED; - - // TODO add more check - if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - if (pOperator->numOfDownstream != 1) { - qError("invalid operator, number of downstream:%d, %s", pOperator->numOfDownstream, id); - return -1; - } - pOperator = pOperator->pDownstream[0]; - } + pOperator = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id); SStreamScanInfo* pInfo = pOperator->info; STableScanInfo* pScanInfo = pInfo->pTableScanOp->info; STableScanBase* pScanBaseInfo = &pScanInfo->base; + STableListInfo* pTableListInfo = pScanBaseInfo->pTableListInfo; if (pOffset->type == TMQ_OFFSET__LOG) { tsdbReaderClose(pScanBaseInfo->dataReader); @@ -1191,9 +1186,13 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } } else { // subType == TOPIC_SUB_TYPE__TABLE/TOPIC_SUB_TYPE__DB + if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; + + SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); + STableListInfo* pTableListInfo = ((STableScanInfo*)(p->info))->base.pTableListInfo; if (setForSnapShot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64" , %s", pOffset->uid, id); return -1; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e29432b808..265f948bcd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -74,25 +74,12 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #endif #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) -#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) - -typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - STableQueryInfo* current; - uint64_t groupId; - SGroupResInfo groupResInfo; - SExprSupp scalarExprSup; - bool groupKeyOptimized; -} SAggOperatorInfo; static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock); static void releaseQueryBuf(size_t numOfTables); -static void destroyAggOperatorInfo(void* param); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); -static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); @@ -266,57 +253,6 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR return pResult; } -// a new buffer page for each table. Needs to opt this design -static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size) { - if (pWindowRes->pageId != -1) { - return 0; - } - - SFilePage* pData = NULL; - - // in the first scan, new space needed for results - int32_t pageId = -1; - SArray* list = getDataBufPagesIdList(pResultBuf); - - if (taosArrayGetSize(list) == 0) { - pData = getNewBufPage(pResultBuf, &pageId); - pData->num = sizeof(SFilePage); - } else { - SPageInfo* pi = getLastPageInfo(list); - pData = getBufPage(pResultBuf, getPageId(pi)); - if (pData == NULL) { - qError("failed to get buffer, code:%s", tstrerror(terrno)); - return terrno; - } - - pageId = getPageId(pi); - - if (pData->num + size > getBufPageSize(pResultBuf)) { - // release current page first, and prepare the next one - releaseBufPageInfo(pResultBuf, pi); - - pData = getNewBufPage(pResultBuf, &pageId); - if (pData != NULL) { - pData->num = sizeof(SFilePage); - } - } - } - - if (pData == NULL) { - return -1; - } - - // set the number of rows in current disk page - if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer - pWindowRes->pageId = pageId; - pWindowRes->offset = (int32_t)pData->num; - - pData->num += size; - } - - return 0; -} - // query_range_start, query_range_end, window_duration, window_start, window_end void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) { pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; @@ -512,25 +448,6 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int return code; } -static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) { - for (int32_t k = 0; k < pOperator->exprSupp.numOfExprs; ++k) { - if (functionNeedToExecute(&pCtx[k])) { - // todo add a dummy funtion to avoid process check - if (pCtx[k].fpSet.process == NULL) { - continue; - } - - int32_t code = pCtx[k].fpSet.process(&pCtx[k]); - if (code != TSDB_CODE_SUCCESS) { - qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); - return code; - } - } - } - - return TSDB_CODE_SUCCESS; -} - bool functionNeedToExecute(SqlFunctionCtx* pCtx) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -654,149 +571,6 @@ STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int return win; } -int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, - uint32_t* status) { - *status = BLK_DATA_NOT_LOAD; - - pBlock->pDataBlock = NULL; - pBlock->pBlockAgg = NULL; - - // int64_t groupId = pRuntimeEnv->current->groupIndex; - // bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - STaskCostInfo* pCost = &pTaskInfo->cost; - -// pCost->totalBlocks += 1; -// pCost->totalRows += pBlock->info.rows; -#if 0 - // Calculate all time windows that are overlapping or contain current data block. - // If current data block is contained by all possible time window, do not load current data block. - if (/*pQueryAttr->pFilters || */pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || - (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pTaskInfo, &pBlock->info))) { - (*status) = BLK_DATA_DATA_LOAD; - } - - // check if this data block is required to load - if ((*status) != BLK_DATA_DATA_LOAD) { - bool needFilter = true; - - // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, - // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - SResultRow* pResult = NULL; - - bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); - TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; - - STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (pQueryAttr->pointInterpQuery) { - needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowEntryInfoOffset); - } else { - if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); - } - } - } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate - doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, - pTableScanInfo->rowEntryInfoOffset, pTableScanInfo->numOfOutput, - pRuntimeEnv->current->groupIndex); - } - - if (needFilter) { - (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); - } else { - (*status) = BLK_DATA_DATA_LOAD; - } - } - - SDataBlockInfo* pBlockInfo = &pBlock->info; -// *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); - - if ((*status) == BLK_DATA_NOT_LOAD || (*status) == BLK_DATA_FILTEROUT) { - //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, -// pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->skipBlocks += 1; - } else if ((*status) == BLK_DATA_SMA_LOAD) { - // this function never returns error? - pCost->loadBlockStatis += 1; -// tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); - - if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); - pCost->totalCheckedRows += pBlock->info.rows; - } - } else { - assert((*status) == BLK_DATA_DATA_LOAD); - - // load the data block statistics to perform further filter - pCost->loadBlockStatis += 1; -// tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); - - if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { - { // set previous window - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - SResultRow* pResult = NULL; - - bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); - TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; - - STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); - } - } - } - bool load = false; - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pTableScanInfo->pCtx[i].functionId; - if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { -// load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockAgg[i].min), -// (char*)&(pBlock->pBlockAgg[i].max)); - if (!load) { // current block has been discard due to filter applied - pCost->skipBlocks += 1; - //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, -// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = BLK_DATA_FILTEROUT; - return TSDB_CODE_SUCCESS; - } - } - } - } - - // current block has been discard due to filter applied -// if (!doFilterByBlockSMA(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { -// pCost->skipBlocks += 1; -// qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, -// pBlockInfo->window.ekey, pBlockInfo->rows); -// (*status) = BLK_DATA_FILTEROUT; -// return TSDB_CODE_SUCCESS; -// } - - pCost->totalCheckedRows += pBlockInfo->rows; - pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); -// if (pBlock->pDataBlock == NULL) { -// return terrno; -// } - -// if (pQueryAttr->pFilters != NULL) { -// filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock); -// } - -// if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { -// filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); -// } - } -#endif - return TSDB_CODE_SUCCESS; -} - void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { if (status == TASK_NOT_COMPLETED) { pTaskInfo->status = status; @@ -1027,43 +801,6 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD } } -void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { - // for simple group by query without interval, all the tables belong to one group result. - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SAggOperatorInfo* pAggInfo = pOperator->info; - - SResultRowInfo* pResultRowInfo = &pAggInfo->binfo.resultRowInfo; - SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset; - - SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, - sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup, true); - /* - * not assign result buffer yet, add new result buffer - * all group belong to one result set, and each group result has different group id so set the id to be one - */ - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize); - if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, terrno); - } - } - - setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); -} - -static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { - SAggOperatorInfo* pAggInfo = pOperator->info; - if (pAggInfo->groupId != UINT64_MAX && pAggInfo->groupId == groupId) { - return; - } - - doSetTableGroupOutputBuf(pOperator, numOfOutput, groupId); - - // record the current active group id - pAggInfo->groupId = groupId; -} - void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowEntryOffset) { bool returnNotNull = false; for (int32_t j = 0; j < numOfExprs; ++j) { @@ -1481,191 +1218,23 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } } -static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) { - if (!tsCountAlwaysReturnValue) { - return TSDB_CODE_SUCCESS; - } - - SAggOperatorInfo* pAggInfo = pOperator->info; - if (pAggInfo->groupKeyOptimized) { - return TSDB_CODE_SUCCESS; - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION || - (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && - ((STableScanInfo*)downstream->info)->hasGroupByTag == true)) { - return TSDB_CODE_SUCCESS; - } - - SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - bool hasCountFunc = false; - - for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; - if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) || - (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) { - hasCountFunc = true; - break; - } - } - - if (!hasCountFunc) { - return TSDB_CODE_SUCCESS; - } - - SSDataBlock* pBlock = createDataBlock(); - pBlock->info.rows = 1; - pBlock->info.capacity = 0; - - for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.hasNull = true; - colInfo.info.type = TSDB_DATA_TYPE_NULL; - colInfo.info.bytes = 1; - - SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i]; - for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { - SFunctParam* pFuncParam = &pOneExpr->base.pParam[j]; - if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { - int32_t slotId = pFuncParam->pCol->slotId; - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - if (slotId >= numOfCols) { - taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1); - for (int32_t k = numOfCols; k < slotId + 1; ++k) { - taosArrayPush(pBlock->pDataBlock, &colInfo); - } - } - } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { - // do nothing - } - } - } - - blockDataEnsureCapacity(pBlock, pBlock->info.rows); - *ppBlock = pBlock; - - return TSDB_CODE_SUCCESS; -} - -static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) { - if (!blockAllocated) { - return; - } - - blockDataDestroy(*ppBlock); - *ppBlock = NULL; -} - -// this is a blocking operator -static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SAggOperatorInfo* pAggInfo = pOperator->info; - - SExprSupp* pSup = &pOperator->exprSupp; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - int64_t st = taosGetTimestampUs(); - - int32_t order = TSDB_ORDER_ASC; - int32_t scanFlag = MAIN_SCAN; - - bool hasValidBlock = false; - bool blockAllocated = false; - - while (1) { - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - if (!hasValidBlock) { - createDataBlockForEmptyInput(pOperator, &pBlock); - if (pBlock == NULL) { - break; - } - blockAllocated = true; - } else { - break; - } - } - hasValidBlock = true; - - int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false); - if (code != TSDB_CODE_SUCCESS) { - destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - T_LONG_JMP(pTaskInfo->env, code); - } - - // there is an scalar expression that needs to be calculated before apply the group aggregation. - if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) { - SExprSupp* pSup1 = &pAggInfo->scalarExprSup; - code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL); - if (code != TSDB_CODE_SUCCESS) { - destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - T_LONG_JMP(pTaskInfo->env, code); - } - } - - // the pDataBlock are always the same one, no need to call this again - setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId); - setInputDataBlock(pSup, pBlock, order, scanFlag, true); - code = doAggregateImpl(pOperator, pSup->pCtx); - if (code != 0) { - destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - T_LONG_JMP(pTaskInfo->env, code); - } - - destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - } - - // the downstream operator may return with error code, so let's check the code before generating results. - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); - } - - initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); - OPTR_SET_OPENED(pOperator); - - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - return pTaskInfo->code; -} - -static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { - SAggOperatorInfo* pAggInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pAggInfo->binfo; - - if (pOperator->status == OP_EXEC_DONE) { +//QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN +SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id) { + if (pOperator == NULL) { + qError("invalid operator, failed to find tableScanOperator %s", id); return NULL; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - setOperatorCompleted(pOperator); - return NULL; - } - - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - while (1) { - doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); - - if (!hasRemainResults(&pAggInfo->groupResInfo)) { - setOperatorCompleted(pOperator); - break; + if (pOperator->operatorType == type) { + return pOperator; + } else { + if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { + qError("invalid operator, failed to find tableScanOperator %s", id); + return NULL; } - if (pInfo->pRes->info.rows > 0) { - break; - } + return extractOperatorInTree(pOperator->pDownstream[0], type, id); } - - size_t rows = blockDataGetNumOfRows(pInfo->pRes); - pOperator->resultInfo.totalRows += rows; - - return (rows == 0) ? NULL : pInfo->pRes; } void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { @@ -1814,7 +1383,7 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock) { initResultRowInfo(&pInfo->resultRowInfo); } -void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { +static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { if (pCtx == NULL) { return NULL; } @@ -1866,99 +1435,8 @@ void cleanupExprSupp(SExprSupp* pSupp) { taosMemoryFree(pSupp->rowEntryInfoOffset); } -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, - SExecTaskInfo* pTaskInfo) { - SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc); - initBasicInfo(&pInfo->binfo, pResBlock); - - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 4096); - - int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); - int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - int32_t numOfScalarExpr = 0; - SExprInfo* pScalarExprInfo = NULL; - if (pAggNode->pExprs != NULL) { - pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); - } - - code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; - pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized; - pInfo->groupId = UINT64_MAX; - - setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, - pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, - optrDefaultBufFn, NULL); - - if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { - STableScanInfo* pTableScanInfo = downstream->info; - pTableScanInfo->base.pdInfo.pExprSup = &pOperator->exprSupp; - pTableScanInfo->base.pdInfo.pAggSup = &pInfo->aggSup; - } - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - if (pInfo != NULL) { - destroyAggOperatorInfo(pInfo); - } - - if (pOperator != NULL) { - cleanupExprSupp(&pOperator->exprSupp); - } - - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -static void freeItem(void* pItem) { - void** p = pItem; - if (*p != NULL) { - taosMemoryFreeClear(*p); - } -} - -void destroyAggOperatorInfo(void* param) { - SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; - cleanupBasicInfo(&pInfo->binfo); - - cleanupAggSup(&pInfo->aggSup); - cleanupExprSupp(&pInfo->scalarExprSup); - cleanupGroupResInfo(&pInfo->groupResInfo); - taosMemoryFreeClear(param); -} - char* buildTaskId(uint64_t taskId, uint64_t queryId) { char* p = taosMemoryMalloc(64); @@ -2147,6 +1625,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); return NULL; } @@ -2154,12 +1633,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo code = extractTableSchemaInfo(pHandle, &pTableScanNode->scan, pTaskInfo); if (code) { pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); return NULL; } pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); if (NULL == pOperator) { pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); return NULL; } @@ -2178,19 +1659,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); return NULL; } - -#ifndef NDEBUG - int32_t sz = tableListGetSize(pTableListInfo); - qDebug("vgId:%d create stream task, total qualified tables:%d, %s", pTaskInfo->id.vgId, sz, idstr); - - for (int32_t i = 0; i < sz; i++) { - STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i); - qDebug("add table uid:%" PRIu64 ", gid:%" PRIu64, pKeyInfo->uid, pKeyInfo->groupId); - } -#endif } pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); @@ -2262,7 +1734,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - if (pOperator != NULL) { + if (pOperator != NULL) { // todo moved away pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; } @@ -2403,9 +1875,7 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return -1; } -int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle) { - SExecTaskInfo* pTask = *(SExecTaskInfo**)pTaskInfo; - +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* pTableListInfo, SReadHandle* readHandle) { switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { SInserterParam* pInserterParam = taosMemoryCalloc(1, sizeof(SInserterParam)); @@ -2422,9 +1892,9 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT if (NULL == pDeleterParam) { return TSDB_CODE_OUT_OF_MEMORY; } -#if 0 - int32_t tbNum = tableListGetSize(pTask->pTableInfoList); - pDeleterParam->suid = tableListGetSuid(pTask->pTableInfoList); + + int32_t tbNum = tableListGetSize(pTableListInfo); + pDeleterParam->suid = tableListGetSuid(pTableListInfo); // TODO extract uid list pDeleterParam->pUidList = taosArrayInit(tbNum, sizeof(uint64_t)); @@ -2434,12 +1904,11 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT } for (int32_t i = 0; i < tbNum; ++i) { - STableKeyInfo* pTable = tableListGetInfo(pTask->pTableInfoList, i); + STableKeyInfo* pTable = tableListGetInfo(pTableListInfo, i); taosArrayPush(pDeleterParam->pUidList, &pTable->uid); } *pParam = pDeleterParam; -#endif break; } @@ -2816,3 +2285,21 @@ void qStreamCloseTsdbReader(void* task) { } } } + +void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { + if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + STableScanInfo* pScanInfo = pOperator->info; + taosArrayPush(pList, &pScanInfo->base.pTableListInfo); + } else { + if (pOperator->pDownstream != NULL && pOperator->pDownstream[0] != NULL) { + extractTableList(pList, pOperator->pDownstream[0]); + } + } +} + +SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) { + SArray* pArray = taosArrayInit(0, POINTER_BYTES); + SOperatorInfo* pOperator = pTaskInfo->pRoot; + extractTableList(pArray, pOperator); + return pArray; +} \ No newline at end of file diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4ce9ed0726..14edf9f13c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -676,7 +676,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } if (pBlock->info.id.uid) { - pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableInfoList, pBlock->info.id.uid); + pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } uint32_t status = 0; @@ -771,7 +771,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { // scan table one by one sequentially if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { - int32_t numOfTables = 0;//tableListGetSize(pTaskInfo->pTableInfoList); + int32_t numOfTables = 0;//tableListGetSize(pTaskInfo->pTableListInfo); STableKeyInfo tInfo = {0}; while (1) { @@ -784,7 +784,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { pInfo->currentTable++; taosRLockLatch(&pTaskInfo->lock); - numOfTables = tableListGetSize(pInfo->base.pTableInfoList); + numOfTables = tableListGetSize(pInfo->base.pTableListInfo); if (pInfo->currentTable >= numOfTables) { qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); @@ -792,7 +792,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return NULL; } - tInfo = *(STableKeyInfo*) tableListGetInfo(pInfo->base.pTableInfoList, pInfo->currentTable); + tInfo = *(STableKeyInfo*) tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1); @@ -804,14 +804,14 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } } else { // scan table group by group sequentially if (pInfo->currentGroupId == -1) { - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableInfoList)) { + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { setOperatorCompleted(pOperator); return NULL; } int32_t num = 0; STableKeyInfo* pList = NULL; - tableListGetGroupList(pInfo->base.pTableInfoList, pInfo->currentGroupId, &pList, &num); + tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, @@ -830,7 +830,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return result; } - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableInfoList)) { + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { setOperatorCompleted(pOperator); return NULL; } @@ -841,7 +841,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - tableListGetGroupList(pInfo->base.pTableInfoList, pInfo->currentGroupId, &pList, &num); + tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); tsdbSetTableList(pInfo->base.dataReader, pList, num); tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); @@ -876,7 +876,7 @@ static void destroyTableScanBase(STableScanBase* pBase) { taosArrayDestroy(pBase->matchInfo.pList); } - tableListDestroy(pBase->pTableInfoList); + tableListDestroy(pBase->pTableListInfo); taosLRUCacheCleanup(pBase->metaCache.pTableMetaEntryCache); cleanupExprSupp(&pBase->pseudoSup); } @@ -945,7 +945,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pInfo->base.pTableInfoList = pTableListInfo; + pInfo->base.pTableListInfo = pTableListInfo; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) { code = terrno; @@ -1065,7 +1065,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU if (hasNext) { /*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); - pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableInfoList, pBlock->info.id.uid); + pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } tsdbReaderClose(pReader); @@ -1087,7 +1087,7 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - return getTableGroupId(pTableScanInfo->base.pTableInfoList, uid); + return getTableGroupId(pTableScanInfo->base.pTableListInfo, uid); } static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { @@ -1562,7 +1562,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock pInfo->pRes->info.version = pBlock->info.version; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - pInfo->pRes->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableInfoList, pBlock->info.id.uid); + pInfo->pRes->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { @@ -2326,6 +2326,7 @@ static void destroyStreamScanOperatorInfo(void* param) { if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) { destroyOperatorInfo(pStreamScan->pTableScanOp); } + if (pStreamScan->tqReader) { tqCloseReader(pStreamScan->tqReader); } @@ -2359,6 +2360,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } @@ -2372,6 +2374,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys int32_t code = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { + tableListDestroy(pTableListInfo); goto _error; } @@ -2391,11 +2394,14 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SExprInfo* pSubTableExpr = taosMemoryCalloc(1, sizeof(SExprInfo)); if (pSubTableExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } + pInfo->tbnameCalSup.pExprInfo = pSubTableExpr; createExprFromOneNode(pSubTableExpr, pTableScanNode->pSubtable, 0); if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1) != 0) { + tableListDestroy(pTableListInfo); goto _error; } } @@ -2405,10 +2411,12 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SExprInfo* pTagExpr = createExpr(pTableScanNode->pTags, &numOfTags); if (pTagExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } } @@ -2416,6 +2424,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } @@ -2459,16 +2468,18 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys // set the extract column id to streamHandle tqReaderSetColIdList(pInfo->tqReader, pColIds); - SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableInfoList); + SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; } + taosArrayDestroy(tableIdList); memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->base.cond, sizeof(SQueryTableDataCond)); } else { taosArrayDestroy(pColIds); + tableListDestroy(pTableListInfo); pColIds = NULL; } @@ -2503,7 +2514,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; + __optr_fn_t nextFn = (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) ? doStreamScan : doQueueScan; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL); @@ -2534,7 +2545,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->pRes; blockDataCleanup(pRes); - int32_t size = tableListGetSize(pInfo->pTableInfoList); + int32_t size = tableListGetSize(pInfo->pTableListInfo); if (size == 0) { setTaskStatus(pTaskInfo, TASK_COMPLETED); return NULL; @@ -2546,7 +2557,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { metaReaderInit(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - STableKeyInfo* item = tableListGetInfo(pInfo->pTableInfoList, pInfo->curPos); + STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); int32_t code = metaGetTableEntryByUid(&mr, item->uid); tDecoderClear(&mr.coder); if (code != TSDB_CODE_SUCCESS) { @@ -2607,7 +2618,7 @@ static void destroyTagScanOperatorInfo(void* param) { STagScanInfo* pInfo = (STagScanInfo*)param; pInfo->pRes = blockDataDestroy(pInfo->pRes); taosArrayDestroy(pInfo->matchInfo.pList); - pInfo->pTableInfoList = tableListDestroy(pInfo->pTableInfoList); + pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2634,7 +2645,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableInfoList = pTableListInfo; + pInfo->pTableListInfo = pTableListInfo; pInfo->pRes = createDataBlockFromDescNode(pDescNode); pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; @@ -2668,7 +2679,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx); int64_t st = taosGetTimestampUs(); - void* p = tableListGetInfo(pInfo->base.pTableInfoList, readIdx + pInfo->tableStartIndex); + void* p = tableListGetInfo(pInfo->base.pTableListInfo, readIdx + pInfo->tableStartIndex); SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { @@ -2725,7 +2736,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { continue; } - pBlock->info.id.groupId = getTableGroupId(pInfo->base.pTableInfoList, pBlock->info.id.uid); + pBlock->info.id.groupId = getTableGroupId(pInfo->base.pTableListInfo, pBlock->info.id.uid); pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; @@ -2781,10 +2792,10 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; { - size_t numOfTables = tableListGetSize(pInfo->base.pTableInfoList); + size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); int32_t i = pInfo->tableStartIndex + 1; for (; i < numOfTables; ++i) { - STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableInfoList, i); + STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i); if (tableKeyInfo->groupId != pInfo->groupId) { break; } @@ -2918,7 +2929,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - size_t tableListSize = tableListGetSize(pInfo->base.pTableInfoList); + size_t tableListSize = tableListGetSize(pInfo->base.pTableListInfo); if (!pInfo->hasGroupId) { pInfo->hasGroupId = true; @@ -2927,7 +2938,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { return NULL; } pInfo->tableStartIndex = 0; - pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pInfo->base.pTableInfoList, pInfo->tableStartIndex))->groupId; + pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex))->groupId; startGroupTableMergeScan(pOperator); } @@ -2952,7 +2963,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = tableListGetInfo(pInfo->base.pTableInfoList, pInfo->tableStartIndex)->groupId; + pInfo->groupId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex)->groupId; startGroupTableMergeScan(pOperator); resetLimitInfoForNextGroup(&pInfo->limitInfo); } @@ -3050,7 +3061,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->base.limitInfo.limit.limit = -1; pInfo->base.limitInfo.slimit.limit = -1; - pInfo->base.pTableInfoList = pTableListInfo; + pInfo->base.pTableListInfo = pTableListInfo; pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 70febcaf6a..4a52683c3a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2175,13 +2175,6 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, S } } -bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { - SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId); - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); - return p1 == NULL; -} - bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) { if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { SWinKey key = {.ts = pWin->skey, .groupId = groupId}; From 6e0f1cb1e012961236778747e86e68769851765b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Apr 2023 11:57:43 +0800 Subject: [PATCH 128/176] refactor: do some internal refactor. --- source/libs/executor/src/aggregateoperator.c | 427 +++++++++++++++++++ 1 file changed, 427 insertions(+) create mode 100644 source/libs/executor/src/aggregateoperator.c diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c new file mode 100644 index 0000000000..565a1ccf25 --- /dev/null +++ b/source/libs/executor/src/aggregateoperator.c @@ -0,0 +1,427 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "tfill.h" +#include "tname.h" + +#include "tdatablock.h" +#include "tglobal.h" +#include "tmsg.h" +#include "ttime.h" + +#include "executorimpl.h" +#include "index.h" +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +static void destroyAggOperatorInfo(void* param); +static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); + +static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock); +static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock); + +static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator); +static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx); +static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator); + +static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size); + +static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); + +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, + SExecTaskInfo* pTaskInfo) { + SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(&pOperator->resultInfo, 4096); + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); + int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + int32_t numOfScalarExpr = 0; + SExprInfo* pScalarExprInfo = NULL; + if (pAggNode->pExprs != NULL) { + pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); + } + + code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; + pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized; + pInfo->groupId = UINT64_MAX; + + setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, + pTaskInfo); + pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, + optrDefaultBufFn, NULL); + + if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + STableScanInfo* pTableScanInfo = downstream->info; + pTableScanInfo->base.pdInfo.pExprSup = &pOperator->exprSupp; + pTableScanInfo->base.pdInfo.pAggSup = &pInfo->aggSup; + } + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + if (pInfo != NULL) { + destroyAggOperatorInfo(pInfo); + } + + if (pOperator != NULL) { + cleanupExprSupp(&pOperator->exprSupp); + } + + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroyAggOperatorInfo(void* param) { + SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; + cleanupBasicInfo(&pInfo->binfo); + + cleanupAggSup(&pInfo->aggSup); + cleanupExprSupp(&pInfo->scalarExprSup); + cleanupGroupResInfo(&pInfo->groupResInfo); + taosMemoryFreeClear(param); +} + +// this is a blocking operator +int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SAggOperatorInfo* pAggInfo = pOperator->info; + + SExprSupp* pSup = &pOperator->exprSupp; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + int64_t st = taosGetTimestampUs(); + + int32_t order = TSDB_ORDER_ASC; + int32_t scanFlag = MAIN_SCAN; + + bool hasValidBlock = false; + bool blockAllocated = false; + + while (1) { + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + if (!hasValidBlock) { + createDataBlockForEmptyInput(pOperator, &pBlock); + if (pBlock == NULL) { + break; + } + blockAllocated = true; + } else { + break; + } + } + hasValidBlock = true; + + int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false); + if (code != TSDB_CODE_SUCCESS) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + T_LONG_JMP(pTaskInfo->env, code); + } + + // there is an scalar expression that needs to be calculated before apply the group aggregation. + if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) { + SExprSupp* pSup1 = &pAggInfo->scalarExprSup; + code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL); + if (code != TSDB_CODE_SUCCESS) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + T_LONG_JMP(pTaskInfo->env, code); + } + } + + // the pDataBlock are always the same one, no need to call this again + setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId); + setInputDataBlock(pSup, pBlock, order, scanFlag, true); + code = doAggregateImpl(pOperator, pSup->pCtx); + if (code != 0) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + T_LONG_JMP(pTaskInfo->env, code); + } + + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + } + + // the downstream operator may return with error code, so let's check the code before generating results. + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } + + initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); + OPTR_SET_OPENED(pOperator); + + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + return pTaskInfo->code; +} + +SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + setOperatorCompleted(pOperator); + return NULL; + } + + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + while (1) { + doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + + if (!hasRemainResults(&pAggInfo->groupResInfo)) { + setOperatorCompleted(pOperator); + break; + } + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + + size_t rows = blockDataGetNumOfRows(pInfo->pRes); + pOperator->resultInfo.totalRows += rows; + + return (rows == 0) ? NULL : pInfo->pRes; +} + +int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) { + for (int32_t k = 0; k < pOperator->exprSupp.numOfExprs; ++k) { + if (functionNeedToExecute(&pCtx[k])) { + // todo add a dummy funtion to avoid process check + if (pCtx[k].fpSet.process == NULL) { + continue; + } + + int32_t code = pCtx[k].fpSet.process(&pCtx[k]); + if (code != TSDB_CODE_SUCCESS) { + qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); + return code; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) { + if (!tsCountAlwaysReturnValue) { + return TSDB_CODE_SUCCESS; + } + + SAggOperatorInfo* pAggInfo = pOperator->info; + if (pAggInfo->groupKeyOptimized) { + return TSDB_CODE_SUCCESS; + } + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION || + (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && + ((STableScanInfo*)downstream->info)->hasGroupByTag == true)) { + return TSDB_CODE_SUCCESS; + } + + SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + bool hasCountFunc = false; + + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; + if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) || + (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) { + hasCountFunc = true; + break; + } + } + + if (!hasCountFunc) { + return TSDB_CODE_SUCCESS; + } + + SSDataBlock* pBlock = createDataBlock(); + pBlock->info.rows = 1; + pBlock->info.capacity = 0; + + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.hasNull = true; + colInfo.info.type = TSDB_DATA_TYPE_NULL; + colInfo.info.bytes = 1; + + SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i]; + for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { + SFunctParam* pFuncParam = &pOneExpr->base.pParam[j]; + if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { + int32_t slotId = pFuncParam->pCol->slotId; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + if (slotId >= numOfCols) { + taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1); + for (int32_t k = numOfCols; k < slotId + 1; ++k) { + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + } + } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { + // do nothing + } + } + } + + blockDataEnsureCapacity(pBlock, pBlock->info.rows); + *ppBlock = pBlock; + + return TSDB_CODE_SUCCESS; +} + +void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) { + if (!blockAllocated) { + return; + } + + blockDataDestroy(*ppBlock); + *ppBlock = NULL; +} + +void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { + SAggOperatorInfo* pAggInfo = pOperator->info; + if (pAggInfo->groupId != UINT64_MAX && pAggInfo->groupId == groupId) { + return; + } + + doSetTableGroupOutputBuf(pOperator, numOfOutput, groupId); + + // record the current active group id + pAggInfo->groupId = groupId; +} + +void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { + // for simple group by query without interval, all the tables belong to one group result. + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SAggOperatorInfo* pAggInfo = pOperator->info; + + SResultRowInfo* pResultRowInfo = &pAggInfo->binfo.resultRowInfo; + SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset; + + SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, + sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup, true); + /* + * not assign result buffer yet, add new result buffer + * all group belong to one result set, and each group result has different group id so set the id to be one + */ + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize); + if (ret != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + } + + setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); +} + +// a new buffer page for each table. Needs to opt this design +int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size) { + if (pWindowRes->pageId != -1) { + return 0; + } + + SFilePage* pData = NULL; + + // in the first scan, new space needed for results + int32_t pageId = -1; + SArray* list = getDataBufPagesIdList(pResultBuf); + + if (taosArrayGetSize(list) == 0) { + pData = getNewBufPage(pResultBuf, &pageId); + pData->num = sizeof(SFilePage); + } else { + SPageInfo* pi = getLastPageInfo(list); + pData = getBufPage(pResultBuf, getPageId(pi)); + if (pData == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return terrno; + } + + pageId = getPageId(pi); + + if (pData->num + size > getBufPageSize(pResultBuf)) { + // release current page first, and prepare the next one + releaseBufPageInfo(pResultBuf, pi); + + pData = getNewBufPage(pResultBuf, &pageId); + if (pData != NULL) { + pData->num = sizeof(SFilePage); + } + } + } + + if (pData == NULL) { + return -1; + } + + // set the number of rows in current disk page + if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer + pWindowRes->pageId = pageId; + pWindowRes->offset = (int32_t)pData->num; + + pData->num += size; + } + + return 0; +} From 81c3d16a2d069aeb4153b56d1036aef9dd356ce9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Apr 2023 13:29:11 +0800 Subject: [PATCH 129/176] fix(query):set the table list ptr. --- source/libs/executor/src/cachescanoperator.c | 2 ++ source/libs/executor/src/sysscanoperator.c | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 0588b9c53a..f6fc332b37 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -54,9 +54,11 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } + pInfo->pTableList = pTableListInfo; pInfo->readHandle = *readHandle; SDataBlockDescNode* pDescNode = pScanNode->scan.node.pOutputDataBlockDesc; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 161811531a..1abe678ac6 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2215,6 +2215,7 @@ static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); tsdbReaderClose(pDistInfo->pHandle); + tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2265,8 +2266,8 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi } pInfo->pTableListInfo = pTableListInfo; - size_t num = tableListGetSize(pTableListInfo); - void* pList = tableListGetInfo(pTableListInfo, 0); + size_t num = tableListGetSize(pTableListInfo); + void* pList = tableListGetInfo(pTableListInfo, 0); code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false); cleanupQueryTableDataCond(&cond); From c5632ca3558a57d7d1c35ca63d121e2ea074aa05 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 3 Apr 2023 13:47:28 +0800 Subject: [PATCH 130/176] enh: convert the month and year in interval-offset to a database precision duration --- tests/script/tsim/query/interval-offset.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index f4d95df083..0d796af0a0 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -212,10 +212,10 @@ print ===> rows2: $data20 $data21 $data22 $data23 $data24 if $rows != 3 then return -1 endi -if $data01 != 2 then +if $data01 != 4 then return -1 endi -if $data04 != 2 then +if $data04 != 4 then return -1 endi From 08bd021a6ff3b9209a1b6ac5318a310cc19ffea6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Apr 2023 14:49:14 +0800 Subject: [PATCH 131/176] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 9 +- source/libs/executor/src/aggregateoperator.c | 142 ++++++++- source/libs/executor/src/executor.c | 3 +- source/libs/executor/src/executorimpl.c | 295 +----------------- source/libs/executor/src/scanoperator.c | 8 +- source/libs/executor/src/timewindowoperator.c | 1 + 6 files changed, 155 insertions(+), 303 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 67c9db13ed..a6353f722a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -485,10 +485,11 @@ typedef struct SStreamScanInfo { } SStreamScanInfo; typedef struct { - SVnode* vnode; - SSDataBlock pRes; // result SSDataBlock - STsdbReader* dataReader; - SSnapContext* sContext; + SVnode* vnode; + SSDataBlock pRes; // result SSDataBlock + STsdbReader* dataReader; + SSnapContext* sContext; + STableListInfo* pTableListInfo; } SStreamRawScanInfo; typedef struct STableCountScanSupp { diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index 565a1ccf25..a26e3ace7b 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -34,6 +34,12 @@ #include "ttypes.h" #include "vnode.h" +typedef struct { + bool hasAgg; + int32_t numOfRows; + int32_t startOffset; +} SFunctionCtxStatus; + static void destroyAggOperatorInfo(void* param); static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); @@ -44,10 +50,16 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator); static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx); static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator); -static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size); +static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, + const char* pKey); + +static int32_t addNewResultRowBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size); static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); +static void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus); +static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus); + SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo) { SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); @@ -366,7 +378,7 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize); + int32_t ret = addNewResultRowBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize); if (ret != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, terrno); } @@ -376,7 +388,7 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin } // a new buffer page for each table. Needs to opt this design -int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size) { +int32_t addNewResultRowBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size) { if (pWindowRes->pageId != -1) { return 0; } @@ -425,3 +437,127 @@ int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, return 0; } + +int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, + const char* pKey) { + int32_t code = 0; + // _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + + pAggSup->currentPageId = -1; + pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); + pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); + pAggSup->pResultRowHashTable = tSimpleHashInit(100, taosFastHash); + + if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + uint32_t defaultPgsz = 0; + uint32_t defaultBufsz = 0; + getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz); + + if (!osTempSpaceAvailable()) { + code = TSDB_CODE_NO_AVAIL_DISK; + qError("Init stream agg supporter failed since %s, %s", terrstr(code), pKey); + return code; + } + + code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir); + if (code != TSDB_CODE_SUCCESS) { + qError("Create agg result buf failed since %s, %s", tstrerror(code), pKey); + return code; + } + + return code; +} + +void cleanupAggSup(SAggSupporter* pAggSup) { + taosMemoryFreeClear(pAggSup->keyBuf); + tSimpleHashCleanup(pAggSup->pResultRowHashTable); + destroyDiskbasedBuf(pAggSup->pResultBuf); +} + +int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, + const char* pkey, void* pState) { + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + for (int32_t i = 0; i < numOfCols; ++i) { + if (pState) { + pSup->pCtx[i].saveHandle.pBuf = NULL; + pSup->pCtx[i].saveHandle.pState = pState; + pSup->pCtx[i].exprIdx = i; + } else { + pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf; + } + } + + return TSDB_CODE_SUCCESS; +} + +void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, + int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) { + for (int32_t k = 0; k < numOfOutput; ++k) { + // keep it temporarily + SFunctionCtxStatus status = {0}; + functionCtxSave(&pCtx[k], &status); + + pCtx[k].input.startRowIndex = offset; + pCtx[k].input.numOfRows = forwardStep; + + // not a whole block involved in query processing, statistics data can not be used + // NOTE: the original value of isSet have been changed here + if (pCtx[k].input.colDataSMAIsSet && forwardStep < numOfTotal) { + pCtx[k].input.colDataSMAIsSet = false; + } + + if (pCtx[k].isPseudoFunc) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]); + + char* p = GET_ROWCELL_INTERBUF(pEntryInfo); + + SColumnInfoData idata = {0}; + idata.info.type = TSDB_DATA_TYPE_BIGINT; + idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + idata.pData = p; + + SScalarParam out = {.columnData = &idata}; + SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; + pCtx[k].sfp.process(&tw, 1, &out); + pEntryInfo->numOfRes = 1; + } else { + int32_t code = TSDB_CODE_SUCCESS; + if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { + code = pCtx[k].fpSet.process(&pCtx[k]); + + if (code != TSDB_CODE_SUCCESS) { + qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code)); + taskInfo->code = code; + T_LONG_JMP(taskInfo->env, code); + } + } + + // restore it + functionCtxRestore(&pCtx[k], &status); + } + } +} + +void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { + pStatus->hasAgg = pCtx->input.colDataSMAIsSet; + pStatus->numOfRows = pCtx->input.numOfRows; + pStatus->startOffset = pCtx->input.startRowIndex; +} + +void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { + pCtx->input.colDataSMAIsSet = pStatus->hasAgg; + pCtx->input.numOfRows = pStatus->numOfRows; + pCtx->input.startRowIndex = pStatus->startOffset; +} \ No newline at end of file diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b05d01f596..e1cd509cba 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1192,7 +1192,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SSnapContext* sContext = pInfo->sContext; SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); - STableListInfo* pTableListInfo = ((STableScanInfo*)(p->info))->base.pTableListInfo; + STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo; + if (setForSnapShot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64" , %s", pOffset->uid, id); return -1; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 265f948bcd..c120c6055d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -34,9 +34,7 @@ #include "ttypes.h" #include "vnode.h" -#define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) - #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #if 0 @@ -81,14 +79,14 @@ static void releaseQueryBuf(size_t numOfTables); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); -static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, - const char* pKey); + static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status); static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo); +static SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; @@ -268,72 +266,6 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataSetInt64(pColData, 4, &pQueryWindow->ekey); } -typedef struct { - bool hasAgg; - int32_t numOfRows; - int32_t startOffset; -} SFunctionCtxStatus; - -static void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { - pStatus->hasAgg = pCtx->input.colDataSMAIsSet; - pStatus->numOfRows = pCtx->input.numOfRows; - pStatus->startOffset = pCtx->input.startRowIndex; -} - -static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { - pCtx->input.colDataSMAIsSet = pStatus->hasAgg; - pCtx->input.numOfRows = pStatus->numOfRows; - pCtx->input.startRowIndex = pStatus->startOffset; -} - -void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, - int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) { - for (int32_t k = 0; k < numOfOutput; ++k) { - // keep it temporarily - SFunctionCtxStatus status = {0}; - functionCtxSave(&pCtx[k], &status); - - pCtx[k].input.startRowIndex = offset; - pCtx[k].input.numOfRows = forwardStep; - - // not a whole block involved in query processing, statistics data can not be used - // NOTE: the original value of isSet have been changed here - if (pCtx[k].input.colDataSMAIsSet && forwardStep < numOfTotal) { - pCtx[k].input.colDataSMAIsSet = false; - } - - if (pCtx[k].isPseudoFunc) { - SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]); - - char* p = GET_ROWCELL_INTERBUF(pEntryInfo); - - SColumnInfoData idata = {0}; - idata.info.type = TSDB_DATA_TYPE_BIGINT; - idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - idata.pData = p; - - SScalarParam out = {.columnData = &idata}; - SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; - pCtx[k].sfp.process(&tw, 1, &out); - pEntryInfo->numOfRes = 1; - } else { - int32_t code = TSDB_CODE_SUCCESS; - if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { - code = pCtx[k].fpSet.process(&pCtx[k]); - - if (code != TSDB_CODE_SUCCESS) { - qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code)); - taskInfo->code = code; - T_LONG_JMP(taskInfo->env, code); - } - } - - // restore it - functionCtxRestore(&pCtx[k], &status); - } - } -} - static void doSetInputDataBlockInfo(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) { SqlFunctionCtx* pCtx = pExprSup->pCtx; for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) { @@ -1018,161 +950,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG } } -// static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, -// STableQueryInfo* pTableQueryInfo) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; -// -// STimeWindow tw = *win; -// getNextTimeWindow(pQueryAttr, &tw); -// -// if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || -// (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { -// -// // load the data block and check data remaining in current data block -// // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// tw = *win; -// int32_t startPos = -// getNextQualifiedWindow(pQueryAttr, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); -// -// // set the abort info -// pQueryAttr->pos = startPos; -// -// // reset the query start timestamp -// pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; -// pQueryAttr->window.skey = pTableQueryInfo->win.skey; -// TSKEY key = pTableQueryInfo->win.skey; -// -// pWindowResInfo->prevSKey = tw.skey; -// int32_t index = pRuntimeEnv->resultRowInfo.curIndex; -// -// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index -// -// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, -// lastKey:%" PRId64, -// GET_TASKID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, -// pQueryAttr->current->lastKey); -// -// return key; -// } else { // do nothing -// pQueryAttr->window.skey = tw.skey; -// pWindowResInfo->prevSKey = tw.skey; -// pTableQueryInfo->lastKey = tw.skey; -// -// return tw.skey; -// } -// -// return true; -// } - -// static bool skipTimeInterval(STaskRuntimeEnv *pRuntimeEnv, TSKEY* start) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// // if queried with value filter, do NOT forward query start position -// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || -// pRuntimeEnv->pFillInfo != NULL) { -// return true; -// } -// -// /* -// * 1. for interval without interpolation query we forward pQueryAttr->interval.interval at a time for -// * pQueryAttr->limit.offset times. Since hole exists, pQueryAttr->interval.interval*pQueryAttr->limit.offset -// value is -// * not valid. otherwise, we only forward pQueryAttr->limit.offset number of points -// */ -// STimeWindow w = TSWINDOW_INITIALIZER; -// bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); -// -// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; -// STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; -// -// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); -// -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { -// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.skey, blockInfo.window.skey, pQueryAttr->window.ekey, -// &w); pWindowResInfo->prevSKey = w.skey; -// } -// } else { -// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.ekey, pQueryAttr->window.ekey, blockInfo.window.ekey, &w); -// pWindowResInfo->prevSKey = w.skey; -// } -// -// // the first time window -// STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQueryAttr); -// -// while (pQueryAttr->limit.offset > 0) { -// STimeWindow tw = win; -// -// if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { -// pQueryAttr->limit.offset -= 1; -// pWindowResInfo->prevSKey = win.skey; -// -// // current time window is aligned with blockInfo.window.ekey -// // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; -// if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { -// pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; -// } -// } -// -// if (pQueryAttr->limit.offset == 0) { -// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); -// return true; -// } -// -// // current window does not ended in current data block, try next data block -// getNextTimeWindow(pQueryAttr, &tw); -// -// /* -// * If the next time window still starts from current data block, -// * load the primary timestamp column first, and then find the start position for the next queried time window. -// * Note that only the primary timestamp column is required. -// * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually -// required -// * time window resides in current data block. -// */ -// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { -// -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { -// pQueryAttr->limit.offset -= 1; -// } -// -// if (pQueryAttr->limit.offset == 0) { -// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); -// return true; -// } else { -// tw = win; -// int32_t startPos = -// getNextQualifiedWindow(pQueryAttr, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); -// // set the abort info -// pQueryAttr->pos = startPos; -// pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; -// pWindowResInfo->prevSKey = tw.skey; -// win = tw; -// } -// } else { -// break; // offset is not 0, and next time window begins or ends in the next block. -// } -// } -// } -// -// // check for error -// if (terrno != TSDB_CODE_SUCCESS) { -// T_LONG_JMP(pRuntimeEnv->env, terrno); -// } -// -// return true; -// } - int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num) { p->pDownstream = taosMemoryCalloc(1, num * POINTER_BYTES); if (p->pDownstream == NULL) { @@ -1301,70 +1078,6 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul return 0; } -int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, - const char* pKey) { - int32_t code = 0; - // _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - - pAggSup->currentPageId = -1; - pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); - pAggSup->pResultRowHashTable = tSimpleHashInit(100, taosFastHash); - - if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - uint32_t defaultPgsz = 0; - uint32_t defaultBufsz = 0; - getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz); - - if (!osTempSpaceAvailable()) { - code = TSDB_CODE_NO_AVAIL_DISK; - qError("Init stream agg supporter failed since %s, %s", terrstr(code), pKey); - return code; - } - - code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir); - if (code != TSDB_CODE_SUCCESS) { - qError("Create agg result buf failed since %s, %s", tstrerror(code), pKey); - return code; - } - - return code; -} - -void cleanupAggSup(SAggSupporter* pAggSup) { - taosMemoryFreeClear(pAggSup->keyBuf); - tSimpleHashCleanup(pAggSup->pResultRowHashTable); - destroyDiskbasedBuf(pAggSup->pResultBuf); -} - -int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey, void* pState) { - int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - for (int32_t i = 0; i < numOfCols; ++i) { - if (pState) { - pSup->pCtx[i].saveHandle.pBuf = NULL; - pSup->pCtx[i].saveHandle.pState = pState; - pSup->pCtx[i].exprIdx = i; - } else { - pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf; - } - } - - return TSDB_CODE_SUCCESS; -} - void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows) { if (numOfRows == 0) { numOfRows = 4096; @@ -1474,8 +1187,6 @@ SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t v return pTaskInfo; } -SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); - int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); @@ -2286,7 +1997,7 @@ void qStreamCloseTsdbReader(void* task) { } } -void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { +static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pScanInfo = pOperator->info; taosArrayPush(pList, &pScanInfo->base.pTableListInfo); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 14edf9f13c..1bfa8ccfc4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2172,19 +2172,19 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { code = tsdbNextDataBlock(pInfo->dataReader, &hasNext); if (code) { tsdbReleaseDataBlock(pInfo->dataReader); - longjmp(pTaskInfo->env, code); + T_LONG_JMP(pTaskInfo->env, code); } } if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { tsdbReleaseDataBlock(pInfo->dataReader); - longjmp(pTaskInfo->env, pTaskInfo->code); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); if (pBlock == NULL) { - longjmp(pTaskInfo->env, terrno); + T_LONG_JMP(pTaskInfo->env, terrno); } qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.id.uid); @@ -2283,6 +2283,7 @@ static void destroyRawScanOperatorInfo(void* param) { SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; tsdbReaderClose(pRawScan->dataReader); destroySnapContext(pRawScan->sContext); + tableListDestroy(pRawScan->pTableListInfo); taosMemoryFree(pRawScan); } @@ -2304,6 +2305,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT goto _end; } + pInfo->pTableListInfo = tableListCreate(); pInfo->vnode = pHandle->vnode; pInfo->sContext = pHandle->sContext; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 4a52683c3a..29acc1fea2 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -938,6 +938,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } + TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); From aa6a4efd5b8b4bdcca6889c973c51864aed2af47 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 3 Apr 2023 16:18:16 +0800 Subject: [PATCH 132/176] fix(query): fix interp tsdbReader external range not setting properly --- source/dnode/vnode/src/tsdb/tsdbRead.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 96bce02b67..836fd09bda 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4001,10 +4001,6 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) { STimeWindow window = pCond->twindows; - if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { - pCond->twindows.skey += 1; - pCond->twindows.ekey -= 1; - } int32_t capacity = pVnode->config.tsdbCfg.maxRows; if (pResBlock != NULL) { @@ -4027,11 +4023,11 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL // update the SQueryTableDataCond to create inner reader int32_t order = pCond->order; if (order == TSDB_ORDER_ASC) { - pCond->twindows.ekey = window.skey; + pCond->twindows.ekey = window.skey - 1; pCond->twindows.skey = INT64_MIN; pCond->order = TSDB_ORDER_DESC; } else { - pCond->twindows.skey = window.ekey; + pCond->twindows.skey = window.ekey + 1; pCond->twindows.ekey = INT64_MAX; pCond->order = TSDB_ORDER_ASC; } @@ -4043,11 +4039,11 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } if (order == TSDB_ORDER_ASC) { - pCond->twindows.skey = window.ekey; + pCond->twindows.skey = window.ekey + 1; pCond->twindows.ekey = INT64_MAX; } else { pCond->twindows.skey = INT64_MIN; - pCond->twindows.ekey = window.ekey; + pCond->twindows.ekey = window.ekey - 1; } pCond->order = order; From 1114d358a403662217ae116ae76081069a3a6337 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 3 Apr 2023 17:06:51 +0800 Subject: [PATCH 133/176] fix:lose consume data because of exec close if consume while insert data --- source/libs/executor/src/projectoperator.c | 28 +++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 97951d3d8a..0f7820f076 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -232,11 +232,11 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } if (pOperator->status == OP_EXEC_DONE) { - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { - pOperator->status = OP_OPENED; - qDebug("projection in queue model, set status open and return null"); - return NULL; - } +// if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { +// pOperator->status = OP_OPENED; +// qDebug("projection in queue model, set status open and return null"); +// return NULL; +// } return NULL; } @@ -262,19 +262,19 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE && pFinalRes->info.rows == 0) { - pOperator->status = OP_OPENED; - return NULL; - } + if (pBlock == NULL && pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) { +// if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE && pFinalRes->info.rows == 0) { +// pOperator->status = OP_OPENED; +// return NULL; +// } qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); setOperatorCompleted(pOperator); break; } - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { - qDebug("set status recv"); - pOperator->status = OP_EXEC_RECV; - } +// if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { +// qDebug("set status recv"); +// pOperator->status = OP_EXEC_RECV; +// } // for stream interval if (pBlock->info.type == STREAM_RETRIEVE || pBlock->info.type == STREAM_DELETE_RESULT || From 3ef7d43dae436b8e2c3238099a8e50ddb3d7662a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Apr 2023 17:47:48 +0800 Subject: [PATCH 134/176] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 38 +------------- source/client/src/clientTmq.c | 14 ++--- source/dnode/vnode/src/tq/tq.c | 20 +++----- source/dnode/vnode/src/tq/tqPush.c | 51 ++++++++++--------- source/dnode/vnode/src/vnd/vnodeSvr.c | 1 + source/libs/executor/inc/executorimpl.h | 20 -------- source/libs/executor/src/aggregateoperator.c | 10 ++++ source/libs/executor/src/executor.c | 6 +-- source/libs/executor/src/scanoperator.c | 11 +++- source/libs/executor/src/timewindowoperator.c | 1 + source/libs/stream/src/stream.c | 49 ++++++++++++++---- source/libs/stream/src/streamData.c | 14 +++-- source/libs/stream/src/streamExec.c | 7 +-- 13 files changed, 115 insertions(+), 127 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index cdcf54bc61..ae48f2fe29 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -371,43 +371,7 @@ SStreamTask* tNewSStreamTask(int64_t streamId); int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask); void tFreeSStreamTask(SStreamTask* pTask); - -static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { - int8_t type = pItem->type; - if (type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); - if (pSubmitClone == NULL) { - qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); - terrno = TSDB_CODE_OUT_OF_MEMORY; - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); - return -1; - } - qDebug("task %d %p submit enqueue %p %p %p %d %" PRId64, pTask->taskId, pTask, pItem, pSubmitClone, - pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, pSubmitClone->submit.ver); - taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); - // qStreamInput(pTask->exec.executor, pSubmitClone); - } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || - type == STREAM_INPUT__REF_DATA_BLOCK) { - taosWriteQitem(pTask->inputQueue->queue, pItem); - // qStreamInput(pTask->exec.executor, pItem); - } else if (type == STREAM_INPUT__CHECKPOINT) { - taosWriteQitem(pTask->inputQueue->queue, pItem); - // qStreamInput(pTask->exec.executor, pItem); - } else if (type == STREAM_INPUT__GET_RES) { - taosWriteQitem(pTask->inputQueue->queue, pItem); - // qStreamInput(pTask->exec.executor, pItem); - } - - if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { - atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); - } - -#if 0 - // TODO: back pressure - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); -#endif - return 0; -} +int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem); static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 59a407656d..71af273243 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -107,7 +107,6 @@ struct tmq_t { STaosQueue* mqueue; // queue of rsp STaosQall* qall; STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit - TdThreadMutex lock; // used to protect the operation on each topic, when updating the epsets. tsem_t rspSem; }; @@ -188,7 +187,6 @@ typedef struct { SMqClientVg* pVg; SMqClientTopic* pTopic; int32_t vgId; - tsem_t rspSem; uint64_t requestId; // request id for debug purpose } SMqPollCbParam; @@ -979,7 +977,6 @@ void tmqFreeImpl(void* handle) { taosFreeQall(tmq->qall); tsem_destroy(&tmq->rspSem); - taosThreadMutexDestroy(&tmq->lock); taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl); taos_close_internal(tmq->pTscObj); @@ -1024,7 +1021,6 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->delayedTask = taosOpenQueue(); pTmq->qall = taosAllocateQall(); - taosThreadMutexInit(&pTmq->lock, NULL); if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL || conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1233,7 +1229,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { - tsem_destroy(&pParam->rspSem); taosMemoryFree(pParam); taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -1419,7 +1414,7 @@ static void freeClientVgInfo(void* param) { taosArrayDestroy(pTopic->vgs); } -static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { +static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { bool set = false; int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); @@ -1471,14 +1466,11 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { taosHashCleanup(pVgOffsetHashMap); - taosThreadMutexLock(&tmq->lock); // destroy current buffered existed topics info if (tmq->clientTopics) { taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo); } - tmq->clientTopics = newTopics; - taosThreadMutexUnlock(&tmq->lock); int8_t flag = (topicNumGet == 0)? TMQ_CONSUMER_STATUS__NO_TOPIC:TMQ_CONSUMER_STATUS__READY; atomic_store_8(&tmq->status, flag); @@ -1742,7 +1734,7 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); + doUpdateLocalEp(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ tDeleteSMqAskEpRsp(rspMsg); *pReset = true; @@ -2163,7 +2155,7 @@ void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* par SMqAskEpRsp rsp; tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &rsp); - tmqUpdateEp(pTmq, head->epoch, &rsp); + doUpdateLocalEp(pTmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d4bdd633e9..ae98589d6c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -819,13 +819,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->pRef = pRef; SReadHandle handle = { - .meta = pVnode->pMeta, - .vnode = pVnode, - .initTableReader = true, - .initTqReader = true, - .version = ver, - }; - + .meta = pVnode->pMeta, .vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { @@ -1393,11 +1387,10 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { - void* pIter = NULL; - bool failed = false; - SStreamDataSubmit2* pSubmit = NULL; + void* pIter = NULL; + bool failed = false; - pSubmit = streamDataSubmitNew(submit); + SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); @@ -1411,7 +1404,10 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { } SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { tqDebug("skip push task %d, task status %d", pTask->taskId, pTask->taskStatus); continue; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 1619829115..c10469b13e 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -207,33 +207,18 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ #endif typedef struct { - void* pKey; + void* pKey; int64_t keyLen; } SItem; static void recordPushedEntry(SArray* cachedKey, void* pIter); +static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq); static void freeItem(void* param) { SItem* p = (SItem*) param; taosMemoryFree(p->pKey); } -static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); - int32_t numOfKeys = (int32_t) taosArrayGetSize(pCachedKeys); - - for (int32_t i = 0; i < numOfKeys; i++) { - SItem* pItem = taosArrayGet(pCachedKeys, i); - if (taosHashRemove(pTq->pPushMgr, pItem->pKey, pItem->keyLen) != 0) { - tqError("vgId:%d, tq push hash remove key error, key: %s", vgId, (char*) pItem->pKey); - } - } - - if (numOfKeys > 0) { - tqDebug("vgId:%d, pushed %d items and remain:%d", vgId, numOfKeys, (int32_t)taosHashGetSize(pTq->pPushMgr)); - } -} - static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int64_t ver, int32_t vgId, char* pData, int32_t dataLen, SArray* pCachedKey) { STqPushEntry* pPushEntry = *(STqPushEntry**)pIter; @@ -347,7 +332,7 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to copy data for stream since out of memory"); + tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", vgId); return -1; } @@ -366,13 +351,6 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v return 0; } -void recordPushedEntry(SArray* cachedKey, void* pIter) { - size_t kLen = 0; - void* key = taosHashGetKey(pIter, &kLen); - SItem item = {.pKey = strndup(key, kLen), .keyLen = kLen}; - taosArrayPush(cachedKey, &item); -} - int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type) { uint64_t consumerId = pRequest->consumerId; @@ -430,3 +408,26 @@ int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64 return 0; } + +void recordPushedEntry(SArray* cachedKey, void* pIter) { + size_t kLen = 0; + void* key = taosHashGetKey(pIter, &kLen); + SItem item = {.pKey = strndup(key, kLen), .keyLen = kLen}; + taosArrayPush(cachedKey, &item); +} + +void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + int32_t numOfKeys = (int32_t) taosArrayGetSize(pCachedKeys); + + for (int32_t i = 0; i < numOfKeys; i++) { + SItem* pItem = taosArrayGet(pCachedKeys, i); + if (taosHashRemove(pTq->pPushMgr, pItem->pKey, pItem->keyLen) != 0) { + tqError("vgId:%d, tq push hash remove key error, key: %s", vgId, (char*) pItem->pKey); + } + } + + if (numOfKeys > 0) { + tqDebug("vgId:%d, pushed %d items and remain:%d", vgId, numOfKeys, (int32_t)taosHashGetSize(pTq->pPushMgr)); + } +} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 718b5979a1..3d2b032156 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -541,6 +541,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return vnodeGetBatchMeta(pVnode, pMsg); case TDMT_VND_TMQ_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); + case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); #if 1 diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a6353f722a..db4c3f0b8d 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -502,32 +502,12 @@ typedef struct STableCountScanSupp { char stbNameFilter[TSDB_TABLE_NAME_LEN]; } STableCountScanSupp; -typedef struct STableCountScanOperatorInfo { - SReadHandle readHandle; - SSDataBlock* pRes; - - STableCountScanSupp supp; - - int32_t currGrpIdx; - SArray* stbUidList; // when group by db_name and/or stable_name -} STableCountScanOperatorInfo; - typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; bool mergeResultBlock; } SOptrBasicInfo; -typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - STableQueryInfo* current; - uint64_t groupId; - SGroupResInfo groupResInfo; - SExprSupp scalarExprSup; - bool groupKeyOptimized; -} SAggOperatorInfo; - typedef struct SIntervalAggOperatorInfo { SOptrBasicInfo binfo; // basic info SAggSupporter aggSup; // aggregate supporter diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index a26e3ace7b..d5fc507b94 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -40,6 +40,16 @@ typedef struct { int32_t startOffset; } SFunctionCtxStatus; +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + STableQueryInfo* current; + uint64_t groupId; + SGroupResInfo groupResInfo; + SExprSupp scalarExprSup; + bool groupKeyOptimized; +} SAggOperatorInfo; + static void destroyAggOperatorInfo(void* param); static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e1cd509cba..5a7ff42ddf 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -127,12 +127,10 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pOperator->status = OP_NOT_OPENED; SStreamScanInfo* pInfo = pOperator->info; - qDebug("stream set total blocks:%d, task id:%s" PRIx64, (int32_t)numOfBlocks, id); - ASSERT(pInfo->validBlockIndex == 0); - ASSERT(taosArrayGetSize(pInfo->pBlockLists) == 0); + qDebug("task stream set total blocks:%d %s", (int32_t)numOfBlocks, id); + ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0); if (type == STREAM_INPUT__MERGED_SUBMIT) { - // ASSERT(numOfBlocks > 1); for (int32_t i = 0; i < numOfBlocks; i++) { SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); taosArrayPush(pInfo->pBlockLists, pReq); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1bfa8ccfc4..e66ec49e77 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -33,7 +33,6 @@ int32_t scanDebug = 0; - #define MULTI_READER_MAX_TABLE_NUM 5000 #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) @@ -52,6 +51,16 @@ typedef struct STableMergeScanSortSourceParam { STsdbReader* dataReader; } STableMergeScanSortSourceParam; +typedef struct STableCountScanOperatorInfo { + SReadHandle readHandle; + SSDataBlock* pRes; + + STableCountScanSupp supp; + + int32_t currGrpIdx; + SArray* stbUidList; // when group by db_name and/or stable_name +} STableCountScanOperatorInfo; + static bool processBlockWithProbability(const SSampleExecInfo* pInfo); bool processBlockWithProbability(const SSampleExecInfo* pInfo) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 29acc1fea2..1a1fb6208d 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -4758,6 +4758,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (!pInfo->pUpdated) { pInfo->pUpdated = taosArrayInit(4, sizeof(SWinKey)); } + if (!pInfo->pUpdatedMap) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pUpdatedMap = tSimpleHashInit(1024, hashFn); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index a4a02b7d65..b4f2600f30 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -92,22 +92,22 @@ int32_t streamSetupTrigger(SStreamTask* pTask) { int32_t streamSchedExec(SStreamTask* pTask) { int8_t schedStatus = atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING); + if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); if (pRunReq == NULL) { atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE); return -1; } + pRunReq->head.vgId = pTask->nodeId; pRunReq->streamId = pTask->streamId; pRunReq->taskId = pTask->taskId; - SRpcMsg msg = { - .msgType = TDMT_STREAM_TASK_RUN, - .pCont = pRunReq, - .contLen = sizeof(SStreamTaskRunReq), - }; + + SRpcMsg msg = { .msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq) }; tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); } + return 0; } @@ -275,7 +275,38 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S return 0; } -// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) { -// // -// return 0; -// } +int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { + int8_t type = pItem->type; + + if (type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); + if (pSubmitClone == NULL) { + qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); + terrno = TSDB_CODE_OUT_OF_MEMORY; + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + return -1; + } + + taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); + qDebug("stream task:%d %p submit enqueue %p %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->taskId, + pTask, pItem, pSubmitClone, pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, + pSubmitClone->submit.ver, pTask->inputQueue->queue->numOfItems); + } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || + type == STREAM_INPUT__REF_DATA_BLOCK) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } else if (type == STREAM_INPUT__CHECKPOINT) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } else if (type == STREAM_INPUT__GET_RES) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } + + if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { + atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); + } + +#if 0 + // TODO: back pressure + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); +#endif + return 0; +} \ No newline at end of file diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 8baebaee42..ddddc1ceb3 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -68,16 +68,20 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); - if (pDataSubmit == NULL) return NULL; + if (pDataSubmit == NULL) { + return NULL; + } + pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); - if (pDataSubmit->dataRef == NULL) goto FAIL; + if (pDataSubmit->dataRef == NULL) { + taosFreeQitem(pDataSubmit); + return NULL; + } + pDataSubmit->submit = submit; *pDataSubmit->dataRef = 1; pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; return pDataSubmit; -FAIL: - taosFreeQitem(pDataSubmit); - return NULL; } SStreamMergedSubmit2* streamMergedSubmitNew() { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 25b2656365..6ef327049c 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -34,7 +34,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; - qDebug("task %d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, + qDebug("stream task:%d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { @@ -268,9 +268,10 @@ int32_t streamExecForAll(SStreamTask* pTask) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt); + qDebug("stream task:%d exec begin, msg batch: %d", pTask->taskId, batchCnt); streamTaskExecImpl(pTask, input, pRes); - qDebug("stream task %d exec end", pTask->taskId); + + qDebug("stream task:%d exec end", pTask->taskId); if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); From 482c319beebaefcb13a7ba6ade6cd7941f396ca1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Apr 2023 18:07:28 +0800 Subject: [PATCH 135/176] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 21 ++----------- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqPush.c | 6 ++-- source/libs/stream/src/stream.c | 26 +++++++++++++--- source/libs/stream/src/streamData.c | 48 +++++++++++++++++------------ 5 files changed, 56 insertions(+), 47 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index ae48f2fe29..0c822a4007 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -223,27 +223,12 @@ static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; } -static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { - int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); - if (dequeueFlag == STREAM_QUEUE__FAILED) { - ASSERT(queue->qItem != NULL); - return streamQueueCurItem(queue); - } else { - queue->qItem = NULL; - taosGetQitem(queue->qall, &queue->qItem); - if (queue->qItem == NULL) { - taosReadAllQitems(queue->queue, queue->qall); - taosGetQitem(queue->qall, &queue->qItem); - } - return streamQueueCurItem(queue); - } -} +void* streamQueueNextItem(SStreamQueue* queue); SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit); +void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit); -void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit); - -SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit); +SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit); typedef struct { char* qmsg; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ae98589d6c..5f171eb5ee 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1431,7 +1431,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { } if (pSubmit) { - streamDataSubmitRefDec(pSubmit); + streamDataSubmitDestroy(pSubmit); taosFreeQitem(pSubmit); } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index c10469b13e..62c80c4ce4 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -30,7 +30,7 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm // update processed atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); streamQueueProcessSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); + streamDataSubmitDestroy(pSubmit); if (pRsp->blockNum > 0) { *ppSubmit = pSubmit; return 0; @@ -58,7 +58,7 @@ int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { } while (pHandle->pushHandle.processedVer > pSubmit->ver + 1) { streamQueueProcessSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); + streamDataSubmitDestroy(pSubmit); pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ); if (pSubmit == NULL) break; } @@ -120,7 +120,7 @@ int32_t tqPreparePush(STQ* pTq, STqHandle* pHandle, int64_t reqId, const SRpcHan int32_t tqEnqueue(STqHandle* pHandle, SStreamDataSubmit* pSubmit) { int8_t inputStatus = atomic_load_8(&pHandle->pushHandle.inputStatus); if (inputStatus == TASK_INPUT_STATUS__NORMAL) { - SStreamDataSubmit* pSubmitClone = streamSubmitRefClone(pSubmit); + SStreamDataSubmit* pSubmitClone = streamSubmitBlockClone(pSubmit); if (pSubmitClone == NULL) { return -1; } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index b4f2600f30..d87d445ba5 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -279,18 +279,18 @@ int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { int8_t type = pItem->type; if (type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); - if (pSubmitClone == NULL) { + SStreamDataSubmit2* pSubmitBlock = streamSubmitBlockClone((SStreamDataSubmit2*)pItem); + if (pSubmitBlock == NULL) { qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } - taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); + taosWriteQitem(pTask->inputQueue->queue, pSubmitBlock); qDebug("stream task:%d %p submit enqueue %p %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->taskId, - pTask, pItem, pSubmitClone, pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, - pSubmitClone->submit.ver, pTask->inputQueue->queue->numOfItems); + pTask, pItem, pSubmitBlock, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen, + pSubmitBlock->submit.ver, pTask->inputQueue->queue->numOfItems); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { taosWriteQitem(pTask->inputQueue->queue, pItem); @@ -309,4 +309,20 @@ int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); #endif return 0; +} + +void* streamQueueNextItem(SStreamQueue* queue) { + int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); + if (dequeueFlag == STREAM_QUEUE__FAILED) { + ASSERT(queue->qItem != NULL); + return streamQueueCurItem(queue); + } else { + queue->qItem = NULL; + taosGetQitem(queue->qall, &queue->qItem); + if (queue->qItem == NULL) { + taosReadAllQitems(queue->queue, queue->qall); + taosGetQitem(queue->qall, &queue->qItem); + } + return streamQueueCurItem(queue); + } } \ No newline at end of file diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index ddddc1ceb3..3fba1cb556 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -48,10 +48,12 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock if (pArray == NULL) { return -1; } + taosArrayPush(pArray, &(SSDataBlock){0}); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); blockDecode(pDataBlock, pRetrieve->data); + // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); @@ -79,25 +81,40 @@ SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { } pDataSubmit->submit = submit; - *pDataSubmit->dataRef = 1; + *pDataSubmit->dataRef = 1; // initialize the reference count to be 1 pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; + return pDataSubmit; } +void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit) { + int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); + ASSERT(ref >= 0 && pDataSubmit->type == STREAM_INPUT__DATA_SUBMIT); + + if (ref == 0) { + taosMemoryFree(pDataSubmit->submit.msgStr); + taosMemoryFree(pDataSubmit->dataRef); + } +} + SStreamMergedSubmit2* streamMergedSubmitNew() { SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)taosAllocateQitem(sizeof(SStreamMergedSubmit2), DEF_QITEM, 0); + if (pMerged == NULL) { + return NULL; + } - if (pMerged == NULL) return NULL; pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); pMerged->dataRefs = taosArrayInit(0, sizeof(void*)); - if (pMerged->dataRefs == NULL || pMerged->submits == NULL) goto FAIL; + + if (pMerged->dataRefs == NULL || pMerged->submits == NULL) { + taosArrayDestroy(pMerged->submits); + taosArrayDestroy(pMerged->dataRefs); + taosFreeQitem(pMerged); + return NULL; + } + pMerged->type = STREAM_INPUT__MERGED_SUBMIT; return pMerged; -FAIL: - if (pMerged->submits) taosArrayDestroy(pMerged->submits); - if (pMerged->dataRefs) taosArrayDestroy(pMerged->dataRefs); - taosFreeQitem(pMerged); - return NULL; } int32_t streamMergeSubmit(SStreamMergedSubmit2* pMerged, SStreamDataSubmit2* pSubmit) { @@ -111,26 +128,17 @@ static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit2* pDataSubmit) atomic_add_fetch_32(pDataSubmit->dataRef, 1); } -SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit) { +SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit) { SStreamDataSubmit2* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); - if (pSubmitClone == NULL) { return NULL; } + streamDataSubmitRefInc(pSubmit); memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit2)); return pSubmitClone; } -void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit) { - int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); - ASSERT(ref >= 0); - if (ref == 0) { - taosMemoryFree(pDataSubmit->submit.msgStr); - taosMemoryFree(pDataSubmit->dataRef); - } -} - SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { ASSERT(elem); if (dst->type == STREAM_INPUT__DATA_BLOCK && elem->type == STREAM_INPUT__DATA_BLOCK) { @@ -168,7 +176,7 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit2*)data); + streamDataSubmitDestroy((SStreamDataSubmit2*)data); taosFreeQitem(data); } else if (type == STREAM_INPUT__MERGED_SUBMIT) { SStreamMergedSubmit2* pMerge = (SStreamMergedSubmit2*)data; From 16394862a6a02d9070d17cbf68463f2e5debb5d2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 3 Apr 2023 18:11:46 +0800 Subject: [PATCH 136/176] enh: last_row is keep order function --- source/libs/function/src/builtins.c | 10 +++++----- source/libs/parser/src/parTranslater.c | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1a039ebda6..a88b1388ba 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -480,14 +480,16 @@ static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t le return code; } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + pFunc->node.resType = + (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; } static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters - pFunc->node.resType = (SDataType){.bytes =tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + pFunc->node.resType = + (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; } @@ -509,13 +511,11 @@ static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; if (!IS_NUMERIC_TYPE(para1Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - for (int32_t i = 1; i < numOfParams; ++i) { SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, i); pValue->notReserved = true; @@ -2491,7 +2491,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last_row", .type = FUNCTION_TYPE_LAST_ROW, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateFirstLast, .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 53816a5436..4541002960 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -755,7 +755,8 @@ static bool isPrimaryKeyImpl(SNode* pExpr) { } else if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { SFunctionNode* pFunc = (SFunctionNode*)pExpr; if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pFunc->funcType || - FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType) { + FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType || + FUNCTION_TYPE_LAST_ROW == pFunc->funcType) { return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0)); } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || FUNCTION_TYPE_IROWTS == pFunc->funcType) { From a3ac8b647cd312f5953bc8d56dd09908cf9b3e18 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Apr 2023 18:20:35 +0800 Subject: [PATCH 137/176] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 2 +- include/util/tqueue.h | 16 ++++++++-------- source/client/src/clientTmq.c | 3 ++- source/client/test/clientTests.cpp | 8 ++++---- source/dnode/vnode/src/tq/tq.c | 25 ++++++++++++------------- source/libs/stream/src/stream.c | 12 +++++++----- 6 files changed, 34 insertions(+), 32 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 0c822a4007..5b1d1fa1bc 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -356,7 +356,7 @@ SStreamTask* tNewSStreamTask(int64_t streamId); int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask); void tFreeSStreamTask(SStreamTask* pTask); -int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem); +int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem); static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 1f6b205cdf..d05b5418b3 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -61,7 +61,7 @@ typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems); typedef struct STaosQnode STaosQnode; -typedef struct STaosQnode { +struct STaosQnode { STaosQnode *next; STaosQueue *queue; int64_t timestamp; @@ -70,9 +70,9 @@ typedef struct STaosQnode { int8_t itype; int8_t reserved[3]; char item[]; -} STaosQnode; +}; -typedef struct STaosQueue { +struct STaosQueue { STaosQnode *head; STaosQnode *tail; STaosQueue *next; // for queue set @@ -84,22 +84,22 @@ typedef struct STaosQueue { int64_t memOfItems; int32_t numOfItems; int64_t threadId; -} STaosQueue; +}; -typedef struct STaosQset { +struct STaosQset { STaosQueue *head; STaosQueue *current; TdThreadMutex mutex; tsem_t sem; int32_t numOfQueues; int32_t numOfItems; -} STaosQset; +}; -typedef struct STaosQall { +struct STaosQall { STaosQnode *current; STaosQnode *start; int32_t numOfItems; -} STaosQall; +}; STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 71af273243..3e963dd3e8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1338,8 +1338,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosWriteQitem(tmq->mqueue, pRspWrapper); + int32_t total = taosQueueItemSize(tmq->mqueue); tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64, - tmq->consumerId, rspType, vgId, tmq->mqueue->numOfItems, requestId); + tmq->consumerId, rspType, vgId, total, requestId); tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 6cb89c13ec..84e3424264 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -161,10 +161,10 @@ void *queryThread(void *arg) { return NULL; } -static int32_t numOfThreads = 1; +int32_t numOfThreads = 1; void tmq_commit_cb_print(tmq_t *pTmq, int32_t code, void *param) { - printf("success, code:%d\n", code); + printf("auto commit success, code:%d\n\n\n\n", code); } void* doConsumeData(void* param) { @@ -173,7 +173,7 @@ void* doConsumeData(void* param) { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - tmq_conf_set(conf, "group.id", "cgrpName12"); + tmq_conf_set(conf, "group.id", "cgrpName41"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); @@ -1060,7 +1060,7 @@ TEST(clientCase, sub_tb_test) { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - tmq_conf_set(conf, "group.id", "cgrpName27"); + tmq_conf_set(conf, "group.id", "cgrpName45"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5f171eb5ee..606cfc294a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1332,7 +1332,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { pRefBlock->dataRef = pRef; atomic_add_fetch_32(pRefBlock->dataRef, 1); - if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { + if (tAppendDataForStream(pTask, (SStreamQueueItem*)pRefBlock) < 0) { qError("stream task input del failed, task id %d", pTask->taskId); atomic_sub_fetch_32(pRef, 1); @@ -1367,7 +1367,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { taosArrayPush(pStreamBlock->blocks, &block); if (!failed) { - if (streamTaskInput(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { + if (tAppendDataForStream(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { qError("stream task input del failed, task id %d", pTask->taskId); continue; } @@ -1388,13 +1388,13 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { void* pIter = NULL; - bool failed = false; + bool succ = true; SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); - failed = true; + succ = false; } while (1) { @@ -1409,20 +1409,19 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { } if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("skip push task %d, task status %d", pTask->taskId, pTask->taskStatus); + tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->taskId, pTask->taskStatus); continue; } - tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, submit.ver); - - if (!failed) { - if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { - tqError("stream task input failed, task id %d", pTask->taskId); + tqDebug("data submit enqueue stream task:%d, ver: %" PRId64, pTask->taskId, submit.ver); + if (succ) { + if (tAppendDataForStream(pTask, (SStreamQueueItem*)pSubmit) < 0) { + tqError("stream task:%d failed to put into queue for, too many", pTask->taskId); continue; } if (streamSchedExec(pTask) < 0) { - tqError("stream task launch failed, task id %d", pTask->taskId); + tqError("stream task:%d launch failed, code:%s", pTask->taskId, tstrerror(terrno)); continue; } } else { @@ -1430,12 +1429,12 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { } } - if (pSubmit) { + if (pSubmit != NULL) { streamDataSubmitDestroy(pSubmit); taosFreeQitem(pSubmit); } - return failed ? -1 : 0; + return succ ? 0 : -1; } int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index d87d445ba5..fd0a5233a1 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -68,7 +68,7 @@ void streamSchedByTimer(void* param, void* tmrId) { atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); - if (streamTaskInput(pTask, (SStreamQueueItem*)trigger) < 0) { + if (tAppendDataForStream(pTask, (SStreamQueueItem*)trigger) < 0) { taosFreeQitem(trigger); taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); return; @@ -123,7 +123,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR /*pData->blocks = pReq->data;*/ /*pBlock->sourceVer = pReq->sourceVer;*/ streamDispatchReqToData(pReq, pData); - if (streamTaskInput(pTask, (SStreamQueueItem*)pData) == 0) { + if (tAppendDataForStream(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; } else { status = TASK_INPUT_STATUS__FAILED; @@ -164,7 +164,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, /*pData->blocks = pReq->data;*/ /*pBlock->sourceVer = pReq->sourceVer;*/ streamRetrieveReqToData(pReq, pData); - if (streamTaskInput(pTask, (SStreamQueueItem*)pData) == 0) { + if (tAppendDataForStream(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; } else { status = TASK_INPUT_STATUS__FAILED; @@ -275,7 +275,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S return 0; } -int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { +int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) { int8_t type = pItem->type; if (type == STREAM_INPUT__DATA_SUBMIT) { @@ -288,9 +288,11 @@ int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { } taosWriteQitem(pTask->inputQueue->queue, pSubmitBlock); + + int32_t total = taosQueueItemSize(pTask->inputQueue->queue); qDebug("stream task:%d %p submit enqueue %p %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->taskId, pTask, pItem, pSubmitBlock, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen, - pSubmitBlock->submit.ver, pTask->inputQueue->queue->numOfItems); + pSubmitBlock->submit.ver, total); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { taosWriteQitem(pTask->inputQueue->queue, pItem); From d48be5c0cea8665a8b5cdc7f42629373101c930a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Apr 2023 18:23:35 +0800 Subject: [PATCH 138/176] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 5 ++++- source/libs/stream/src/stream.c | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 606cfc294a..5893a4b941 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1415,7 +1415,10 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { tqDebug("data submit enqueue stream task:%d, ver: %" PRId64, pTask->taskId, submit.ver); if (succ) { - if (tAppendDataForStream(pTask, (SStreamQueueItem*)pSubmit) < 0) { + int32_t code = tAppendDataForStream(pTask, (SStreamQueueItem*)pSubmit); + if (code < 0) { + // let's handle the back pressure + tqError("stream task:%d failed to put into queue for, too many", pTask->taskId); continue; } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index fd0a5233a1..df8847c26d 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -310,6 +310,7 @@ int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) { // TODO: back pressure atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); #endif + return 0; } From 6a3bcf4a71b4aa3c37dc767e5b3e4274dcff72eb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 3 Apr 2023 19:08:57 +0800 Subject: [PATCH 139/176] ehn(plan/optimizer): remove tbname fetching with cached groupby --- source/libs/planner/src/planOptimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 16a5ef7bae..8fee17d968 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2279,7 +2279,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (NULL != cxt.pLastCols) { cxt.doAgg = false; lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols); - nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt); + NODES_DESTORY_LIST(pScan->pScanPseudoCols); lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols); nodesClearList(cxt.pLastCols); } From d651ba02ae66e4077f36d7a165d17890771f62e3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 3 Apr 2023 19:54:52 +0800 Subject: [PATCH 140/176] fix:lose consume data because of exec close if consume while insert data --- include/libs/executor/executor.h | 2 ++ source/dnode/vnode/src/tq/tqPush.c | 2 +- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executor.c | 8 ++++++-- source/libs/executor/src/projectoperator.c | 15 +-------------- 6 files changed, 12 insertions(+), 19 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 6b993fe9c7..33172a4f86 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -198,6 +198,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); +void qStreamSetOpen(qTaskInfo_t tinfo); + void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 1619829115..1bed07e7d9 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -253,7 +253,7 @@ static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int6 if (qStreamSetScanMemData(pTaskInfo, submit) != 0) { return; } - + qStreamSetOpen(pTaskInfo); // here start to scan submit block to extract the subscribed data int32_t totalRows = 0; diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index dff162d527..b4e50312fd 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -81,7 +81,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; - + qStreamSetOpen(task); tqDebug("consumer:0x%"PRIx64" vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { tqError("consumer:0x%"PRIx64" vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8de29662c8..98fba39950 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -192,7 +192,7 @@ enum { OP_OPENED = 0x1, OP_RES_TO_RETURN = 0x5, OP_EXEC_DONE = 0x9, - OP_EXEC_RECV = 0x11, +// OP_EXEC_RECV = 0x11, }; typedef struct SOperatorFpSet { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fdb2a3da06..8058ffd423 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1074,6 +1074,12 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { return 0; } +void qStreamSetOpen(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + pOperator->status = OP_NOT_OPENED; +} + int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; @@ -1086,8 +1092,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } if (subType == TOPIC_SUB_TYPE__COLUMN) { - pOperator->status = OP_OPENED; - if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream != 1) { qError("invalid operator, number of downstream:%d, %s", pOperator->numOfDownstream, id); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 0f7820f076..86c49e0fc8 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -227,17 +227,8 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { blockDataCleanup(pFinalRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pTaskInfo->streamInfo.submit.msgStr) { - pOperator->status = OP_OPENED; - } if (pOperator->status == OP_EXEC_DONE) { -// if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { -// pOperator->status = OP_OPENED; -// qDebug("projection in queue model, set status open and return null"); -// return NULL; -// } - return NULL; } @@ -262,11 +253,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL && pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) { -// if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE && pFinalRes->info.rows == 0) { -// pOperator->status = OP_OPENED; -// return NULL; -// } + if (pBlock == NULL) { qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); setOperatorCompleted(pOperator); break; From fc5890c8bf07671c21ce2ad51e36f8f740b61661 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 3 Apr 2023 12:36:45 +0000 Subject: [PATCH 141/176] aovid conn leak --- source/libs/transport/src/transCli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 50ed9fa61b..c23d6d0a1f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -462,6 +462,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (transQueueEmpty(&pConn->cliMsgs)) { if (pConn->broken == true && CONN_NO_PERSIST_BY_APP(pConn)) { tTrace("%s conn %p handle except, persist:0", CONN_GET_INST_LABEL(pConn), pConn); + if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); return; } @@ -521,6 +522,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { destroyCmsg(pMsg); tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn)); } while (!transQueueEmpty(&pConn->cliMsgs)); + if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); } void cliHandleExcept(SCliConn* conn) { From 45726cf551887269e33c666ba1ab99bdbbe540b7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 3 Apr 2023 23:39:35 +0800 Subject: [PATCH 142/176] fix: taosbenchmark multithreads with limit for main (#20744) * fix: taosbenchmark multithreads with limit for main * fix: update taos-tools --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index aef89a2d42..2de3881dd2 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG d194dc9 + GIT_TAG 273a3fe SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From f79eeb3032e422cef9676ee65f89c8aabab9c494 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 4 Apr 2023 08:54:28 +0800 Subject: [PATCH 143/176] fix: add stmt error handling --- source/client/inc/clientLog.h | 1 + source/client/inc/clientStmt.h | 4 +++ source/client/src/clientEnv.c | 1 + source/client/src/clientStmt.c | 22 ++++++++++++-- tests/script/api/batchprepare.c | 51 ++++++++++++++++++++++++++++++++- 5 files changed, 75 insertions(+), 4 deletions(-) diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index c29f495201..908e470830 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -26,6 +26,7 @@ extern "C" { #define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0) #define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0) #define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarnL(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLongString("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) #define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 2b42de93e3..0c9696f1c8 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -102,6 +102,7 @@ typedef struct STscStmt { SStmtBindInfo bInfo; int64_t reqid; + int32_t errCode; } STscStmt; extern char *gStmtStatusStr[]; @@ -121,6 +122,7 @@ extern char *gStmtStatusStr[]; int32_t _code = c; \ if (_code != TSDB_CODE_SUCCESS) { \ terrno = _code; \ + pStmt->errCode = _code; \ return _code; \ } \ } while (0) @@ -129,6 +131,7 @@ extern char *gStmtStatusStr[]; int32_t _code = c; \ if (_code != TSDB_CODE_SUCCESS) { \ terrno = _code; \ + pStmt->errCode = _code; \ } \ return _code; \ } while (0) @@ -137,6 +140,7 @@ extern char *gStmtStatusStr[]; code = c; \ if (code != TSDB_CODE_SUCCESS) { \ terrno = code; \ + pStmt->errCode = code; \ goto _return; \ } \ } while (0) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index de08ba66cc..dba1dbcf9a 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -103,6 +103,7 @@ static void deregisterRequest(SRequestObj *pRequest) { if (duration >= SLOW_QUERY_INTERVAL) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); + tscWarnL("slow query: %s, duration:%" PRId64, pRequest->sqlstr, duration); } releaseTscObj(pTscObj->id); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 3ed157efef..71a41c68e7 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -32,8 +32,14 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { STMT_LOG_SEQ(newStatus); } + if (pStmt->errCode && newStatus != STMT_PREPARE) { + STMT_DLOG("stmt already failed with err: %s", tstrerror(pStmt->errCode)); + return pStmt->errCode; + } + switch (newStatus) { case STMT_PREPARE: + pStmt->errCode = 0; break; case STMT_SETTBNAME: if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND) || STMT_STATUS_EQ(BIND_COL)) { @@ -197,7 +203,10 @@ int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHa STscStmt* pStmt = (STscStmt*)stmt; *pVgHash = pStmt->sql.pVgHash; + pStmt->sql.pVgHash = NULL; + *pBlockHash = pStmt->exec.pBlockHash; + pStmt->exec.pBlockHash = NULL; return TSDB_CODE_SUCCESS; } @@ -325,6 +334,8 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { } int32_t stmtCleanSQLInfo(STscStmt* pStmt) { + STMT_DLOG_E("start to free SQL info"); + taosMemoryFree(pStmt->sql.queryRes.fields); taosMemoryFree(pStmt->sql.queryRes.userFields); taosMemoryFree(pStmt->sql.sqlStr); @@ -351,6 +362,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { memset(&pStmt->sql, 0, sizeof(pStmt->sql)); + STMT_DLOG_E("end to free SQL info"); + return TSDB_CODE_SUCCESS; } @@ -441,11 +454,10 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { .mgmtEps = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp)}; int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta); if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { - STMT_ERR_RET(stmtCleanBindInfo(pStmt)); - tscDebug("tb %s not exist", pStmt->bInfo.tbFName); + stmtCleanBindInfo(pStmt); - return TSDB_CODE_SUCCESS; + STMT_ERR_RET(code); } STMT_ERR_RET(code); @@ -922,9 +934,13 @@ _return: int stmtClose(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_DLOG_E("start to free stmt"); + stmtCleanSQLInfo(pStmt); taosMemoryFree(stmt); + STMT_DLOG_E("stmt freed"); + return TSDB_CODE_SUCCESS; } diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 0903095dc9..99507ef5c3 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -122,9 +122,11 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos); int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); +int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos); enum { TTYPE_INSERT = 1, + TTYPE_INSERT_NG, TTYPE_QUERY, }; @@ -187,6 +189,8 @@ CaseCfg gCase[] = { {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:NG-TBNEXISTS",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, false, insertNonExistsTb, 10, 10, 1, 3, 0, 0, 1, -1}, + // {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, // {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, @@ -250,7 +254,7 @@ CaseCtrl gCaseCtrl = { .funcIdxList = NULL, .checkParamNum = false, .runTimes = 0, - .caseIdx = 24, + .caseIdx = 26, .caseNum = 1, .caseRunIdx = -1, .caseRunNum = -1, @@ -2191,6 +2195,47 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { } +int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + char *buf = "tbnexist"; + code = bpSetTableNameTags(&data, 0, buf, stmt); + if (code == 0){ + printf("!!!taos_stmt_set_tbname expected error not occurred\n"); + exit(1); + } + + if (0 == taos_stmt_bind_param_batch(stmt, data.pBind)) { + printf("!!!taos_stmt_bind_param_batch expected error not occurred\n"); + exit(1); + } + + if (0 == taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch expected error not occurred\n"); + exit(1); + } + + if (0 == taos_stmt_execute(stmt)) { + printf("!!!taos_stmt_execute expected error not occurred\n"); + exit(1); + } + + destroyData(&data); + + return 0; +} + + + int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -2213,6 +2258,10 @@ int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { } void prepareCheckResultImpl(TAOS * taos, char *tname, bool printr, int expected, bool silent) { + if (TTYPE_INSERT_NG == gCurCase->testType) { + return; + } + char sql[255] = "SELECT * FROM "; int32_t rows = 0; From d547b9e38f5268ab8272402ea9555a17f35e5a6a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 4 Apr 2023 11:27:54 +0800 Subject: [PATCH 144/176] fix: interp pseudo column can not be used without interp function --- include/libs/nodes/querynodes.h | 1 + source/libs/parser/src/parTranslater.c | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 1a9700907e..480912a8cf 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -298,6 +298,7 @@ typedef struct SSelectStmt { bool hasUniqueFunc; bool hasTailFunc; bool hasInterpFunc; + bool hasInterpPseudoColFunc; bool hasLastRowFunc; bool hasLastFunc; bool hasTimeLineFunc; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 4541002960..8e44edce17 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1691,6 +1691,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType); pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType); pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType); + pSelect->hasInterpPseudoColFunc = pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId); pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType); pSelect->hasLastFunc = pSelect->hasLastFunc ? true : (FUNCTION_TYPE_LAST == pFunc->funcType); pSelect->hasTimeLineFunc = pSelect->hasTimeLineFunc ? true : fmIsTimelineFunc(pFunc->funcId); @@ -3369,6 +3370,9 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL != pSelect->pRange || NULL != pSelect->pEvery || NULL != pSelect->pFill) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE); } + if (pSelect->hasInterpPseudoColFunc) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "Missing interp function"); + } return TSDB_CODE_SUCCESS; } From bf486e99f29f533a275066c48fef12bab7c9789b Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 4 Apr 2023 15:20:35 +0800 Subject: [PATCH 145/176] fix: change eror msg --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8e44edce17..6ddd4b2d36 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3371,7 +3371,7 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE); } if (pSelect->hasInterpPseudoColFunc) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "Missing interp function"); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "Has Interp pseudo column(s) but missing interp function"); } return TSDB_CODE_SUCCESS; } From 7d9befbe3af49ab9b0755c80f053755d71cfa29f Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 4 Apr 2023 15:24:14 +0800 Subject: [PATCH 146/176] fix: add testcase --- tests/system-test/2-query/interp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index d7344c631f..71dab8fe70 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -2040,6 +2040,8 @@ class TDTestCase: #tdSql.query(f"select _irowts,interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:04', '2020-02-02 00:00:16') every(1h) fill(prev)") #tdSql.query(f"select tbname,_irowts,interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:04', '2020-02-02 00:00:16') every(1h) fill(prev)") + tdLog.printNoPrefix("======step 14: test interp pseudo columns") + tdSql.error(f"select _irowts, c6 from {dbname}.{tbname}") def stop(self): tdSql.close() From ed53b9827a92847ae9a20d383e5013921db5684a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 4 Apr 2023 15:57:27 +0800 Subject: [PATCH 147/176] fix:modify tsim cases for msg number --- tests/parallel_test/cases.task | 2 +- tests/script/tsim/tmq/basic1.sim | 18 ++-- tests/script/tsim/tmq/basic1Of2Cons.sim | 42 ++++----- tests/script/tsim/tmq/basic2.sim | 18 ++-- tests/script/tsim/tmq/basic2Of2Cons.sim | 42 ++++----- .../script/tsim/tmq/basic2Of2ConsOverlap.sim | 92 +++++++++---------- tests/script/tsim/tmq/basic3.sim | 24 ++--- tests/script/tsim/tmq/basic3Of2Cons.sim | 40 ++++---- tests/script/tsim/tmq/basic4.sim | 18 ++-- tests/script/tsim/tmq/basic4Of2Cons.sim | 40 ++++---- 10 files changed, 168 insertions(+), 168 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 49f14191b1..d1fbacdadf 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -94,7 +94,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index 6880f290f5..80a05ba45c 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -111,8 +111,8 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -134,7 +134,7 @@ if $data[0][2] != $expectmsgcnt then print expect $expectmsgcnt , actual $data02 return -1 endi -if $data[0][3] != $expectmsgcnt then +if $data[0][3] != $totalMsgOfStb then return -1 endi $loop_cnt = $loop_cnt + 1 @@ -183,8 +183,8 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -202,7 +202,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -254,8 +254,8 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -273,7 +273,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 11b645c4d1..2188341804 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -111,11 +111,11 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -142,14 +142,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then + if $data[1][2] == $expectmsgcnt then goto check_ok_0 endi endi @@ -214,10 +214,10 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -244,14 +244,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi @@ -316,10 +316,10 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -346,14 +346,14 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_4 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_4 endi endi diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim index dce73be592..a965645660 100644 --- a/tests/script/tsim/tmq/basic2.sim +++ b/tests/script/tsim/tmq/basic2.sim @@ -83,8 +83,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -105,7 +105,7 @@ endi if $data[0][2] != $expectmsgcnt then return -1 endi -if $data[0][3] != $expectmsgcnt then +if $data[0][3] != $totalMsgOfStb then return -1 endi @@ -140,8 +140,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -159,7 +159,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -197,8 +197,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -216,7 +216,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index 87559305ba..fd61c4b40f 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -82,10 +82,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -112,14 +112,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then + if $data[1][2] == $expectmsgcnt then goto check_ok_0 endi endi @@ -169,10 +169,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -199,14 +199,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi @@ -256,10 +256,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -286,14 +286,14 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_4 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_4 endi endi diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index dda5e0059e..180c8947db 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -80,8 +80,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 3 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_stb_all @@ -89,7 +89,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_stb_function $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -118,40 +118,40 @@ endi # $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb. -if $data[0][2] < $totalMsgOfOneTopic then +#if $data[0][2] < $totalMsgOfOneTopic then +# return -1 +#endi +if $data[0][2] > $expectmsgcnt then return -1 endi -if $data[0][2] > $totalMsgOfStb then - return -1 -endi -if $data[1][2] < $totalMsgOfOneTopic then - return -1 -endi -if $data[1][2] > $totalMsgOfStb then +#if $data[1][2] < $totalMsgOfOneTopic then +# return -1 +#endi +if $data[1][2] > $expectmsgcnt then return -1 endi -$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb +#$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb $sumOfMsgCnt = $data[0][2] + $data[1][2] -if $sumOfMsgCnt != $totalMsgCons then - print total: $totalMsgCons +if $sumOfMsgCnt != $expectmsgcnt then + print total: $expectmsgcnt print sum: $sumOfMsgCnt return -1 endi # $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb. -if $data[0][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[0][3] > $totalMsgOfStb then - return -1 -endi -if $data[1][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[1][3] > $totalMsgOfStb then - return -1 -endi +#if $data[0][3] < $totalMsgOfStb then +# return -1 +#endi +#if $data[0][3] > $totalMsgOfStb then +# return -1 +#endi +#if $data[1][3] < $totalMsgOfStb then +# return -1 +#endi +#if $data[1][3] > $totalMsgOfStb then +# return -1 +#endi $totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb $sumOfRows = $data[0][3] + $data[1][3] @@ -189,15 +189,15 @@ $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ctb_function $topicList = $topicList . , $topicList = $topicList . topic_ctb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -226,24 +226,24 @@ endi # either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb # or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfCtb then +if $data[0][2] == $topicNum then + if $data[1][2] == 1 then goto check_ok_0 endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfCtb then +elif $data[0][2] == 1 then + if $data[1][2] == $topicNum then goto check_ok_0 endi endi return -1 check_ok_0: -if $data[0][3] == $totalMsgOfOneTopic then - if $data[1][3] == $totalMsgOfCtb then +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == $totalMsgOfOneTopic then goto check_ok_1 endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfCtb then +elif $data[0][3] == $totalMsgOfOneTopic then + if $data[1][3] == $totalMsgOfCtb then goto check_ok_1 endi endi @@ -280,8 +280,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ntb_function @@ -289,7 +289,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_ntb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -318,12 +318,12 @@ endi # either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb # or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfNtb then +if $data[0][2] == $expectmsgcnt then + if $data[1][2] == 1 then goto check_ok_2 endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfNtb then +elif $data[0][2] == 1 then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi @@ -334,8 +334,8 @@ if $data[0][3] == $totalMsgOfOneTopic then if $data[1][3] == $totalMsgOfNtb then goto check_ok_3 endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfNtb then +elif $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == $totalMsgOfOneTopic then goto check_ok_3 endi endi diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim index 8d677766d7..63b3665cd0 100644 --- a/tests/script/tsim/tmq/basic3.sim +++ b/tests/script/tsim/tmq/basic3.sim @@ -111,8 +111,8 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -130,10 +130,10 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then +#if $data[0][2] != $expectmsgcnt then +# return -1 +#endi +if $data[0][3] != $totalMsgOfStb then return -1 endi $loop_cnt = $loop_cnt + 1 @@ -182,8 +182,8 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -201,7 +201,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -253,8 +253,8 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -272,7 +272,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index 4921c86c45..fb3091642c 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -110,10 +110,10 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 3 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -163,19 +163,19 @@ endi if $data[0][3] <= 0 then return -1 endi -if $data[0][3] >= $expectmsgcnt then +if $data[0][3] >= $totalMsgOfStb then return -1 endi if $data[1][3] <= 0 then return -1 endi -if $data[1][3] >= $expectmsgcnt then +if $data[1][3] >= $totalMsgOfStb then return -1 endi $sumOfMsgRows = $data[0][3] + $data[1][3] -if $sumOfMsgRows != $expectmsgcnt then +if $sumOfMsgRows != $totalMsgOfStb then return -1 endi @@ -225,10 +225,10 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -255,13 +255,13 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi -elif $data[1][2] == $totalMsgOfCtb then +elif $data[1][2] == $expectmsgcnt then if $data[0][2] == 0 then goto check_ok_0 endi @@ -327,10 +327,10 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -357,13 +357,13 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi -elif $data[1][2] == $totalMsgOfNtb then +elif $data[1][2] == $expectmsgcnt then if $data[0][2] == 0 then goto check_ok_2 endi diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim index 9b418f12f2..2cac9beea6 100644 --- a/tests/script/tsim/tmq/basic4.sim +++ b/tests/script/tsim/tmq/basic4.sim @@ -80,8 +80,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 9 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -102,7 +102,7 @@ endi if $data[0][2] != $expectmsgcnt then return -1 endi -if $data[0][3] != $expectmsgcnt then +if $data[0][3] != $totalMsgOfStb then return -1 endi @@ -137,8 +137,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -156,7 +156,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -194,8 +194,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -213,7 +213,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index f1755f732b..1d0b8f17a3 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -79,10 +79,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 9 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -131,19 +131,19 @@ endi if $data[0][3] <= 0 then return -1 endi -if $data[0][3] >= $expectmsgcnt then +if $data[0][3] >= $totalMsgOfStb then return -1 endi if $data[1][3] <= 0 then return -1 endi -if $data[1][3] >= $expectmsgcnt then +if $data[1][3] >= $totalMsgOfStb then return -1 endi $sumOfConsRow = $data[0][3] + $data[1][3] -if $sumOfConsRow != $expectmsgcnt then +if $sumOfConsRow != $totalMsgOfStb then return -1 endi @@ -178,10 +178,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -208,14 +208,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_0 endi endi @@ -266,10 +266,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -296,14 +296,14 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi From bd4c33d1c216e1f15bed650b50aa9a67dc3574e5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 4 Apr 2023 15:57:50 +0800 Subject: [PATCH 148/176] fix(tmq):fix the invalid read. --- source/libs/stream/src/stream.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index df8847c26d..361cd2cacc 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -287,12 +287,12 @@ int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem) { return -1; } - taosWriteQitem(pTask->inputQueue->queue, pSubmitBlock); - - int32_t total = taosQueueItemSize(pTask->inputQueue->queue); + int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1; qDebug("stream task:%d %p submit enqueue %p %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->taskId, pTask, pItem, pSubmitBlock, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen, pSubmitBlock->submit.ver, total); + + taosWriteQitem(pTask->inputQueue->queue, pSubmitBlock); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { taosWriteQitem(pTask->inputQueue->queue, pItem); From 9b2a94f281f5e4e209b055aaccb42980b329630b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 4 Apr 2023 18:04:10 +0800 Subject: [PATCH 149/176] fix: column comment is not supported --- include/common/ttokendef.h | 36 +- source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/sql.c | 3592 +++++++++---------- source/libs/parser/test/parInitialCTest.cpp | 10 +- 4 files changed, 1816 insertions(+), 1824 deletions(-) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index a0593e7d4b..f941922711 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -127,24 +127,24 @@ #define TK_NK_EQ 109 #define TK_USING 110 #define TK_TAGS 111 -#define TK_COMMENT 112 -#define TK_BOOL 113 -#define TK_TINYINT 114 -#define TK_SMALLINT 115 -#define TK_INT 116 -#define TK_INTEGER 117 -#define TK_BIGINT 118 -#define TK_FLOAT 119 -#define TK_DOUBLE 120 -#define TK_BINARY 121 -#define TK_NCHAR 122 -#define TK_UNSIGNED 123 -#define TK_JSON 124 -#define TK_VARCHAR 125 -#define TK_MEDIUMBLOB 126 -#define TK_BLOB 127 -#define TK_VARBINARY 128 -#define TK_DECIMAL 129 +#define TK_BOOL 112 +#define TK_TINYINT 113 +#define TK_SMALLINT 114 +#define TK_INT 115 +#define TK_INTEGER 116 +#define TK_BIGINT 117 +#define TK_FLOAT 118 +#define TK_DOUBLE 119 +#define TK_BINARY 120 +#define TK_NCHAR 121 +#define TK_UNSIGNED 122 +#define TK_JSON 123 +#define TK_VARCHAR 124 +#define TK_MEDIUMBLOB 125 +#define TK_BLOB 126 +#define TK_VARBINARY 127 +#define TK_DECIMAL 128 +#define TK_COMMENT 129 #define TK_MAX_DELAY 130 #define TK_WATERMARK 131 #define TK_ROLLUP 132 diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 9fd8d5415a..301d3fb982 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -330,7 +330,7 @@ column_def_list(A) ::= column_def(B). column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); } column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); } -column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } +//column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } %type type_name { SDataType } %destructor type_name { } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 291d35ebe3..123aada809 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -139,17 +139,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 752 -#define YYNRULE 572 +#define YYNSTATE 750 +#define YYNRULE 571 #define YYNTOKEN 328 -#define YY_MAX_SHIFT 751 -#define YY_MIN_SHIFTREDUCE 1118 -#define YY_MAX_SHIFTREDUCE 1689 -#define YY_ERROR_ACTION 1690 -#define YY_ACCEPT_ACTION 1691 -#define YY_NO_ACTION 1692 -#define YY_MIN_REDUCE 1693 -#define YY_MAX_REDUCE 2264 +#define YY_MAX_SHIFT 749 +#define YY_MIN_SHIFTREDUCE 1116 +#define YY_MAX_SHIFTREDUCE 1686 +#define YY_ERROR_ACTION 1687 +#define YY_ACCEPT_ACTION 1688 +#define YY_NO_ACTION 1689 +#define YY_MIN_REDUCE 1690 +#define YY_MAX_REDUCE 2260 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -218,273 +218,273 @@ typedef union { *********** Begin parsing tables **********************************************/ #define YY_ACTTAB_COUNT (2669) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 1975, 1845, 387, 2076, 429, 604, 627, 2062, 141, 2235, - /* 10 */ 161, 2135, 45, 43, 1617, 1973, 628, 363, 2058, 1858, - /* 20 */ 384, 2062, 1466, 616, 603, 182, 1907, 2062, 496, 2236, - /* 30 */ 605, 2030, 2058, 1547, 492, 1464, 2094, 640, 2058, 44, - /* 40 */ 42, 41, 40, 39, 619, 1170, 332, 1169, 640, 2044, - /* 50 */ 1491, 657, 2054, 2060, 365, 506, 139, 1968, 1542, 1909, - /* 60 */ 222, 2240, 1909, 651, 18, 2235, 2054, 2060, 366, 349, - /* 70 */ 1691, 1472, 2054, 2060, 379, 1908, 1171, 651, 1907, 404, - /* 80 */ 2075, 2239, 176, 651, 2111, 2236, 2238, 109, 2077, 661, - /* 90 */ 2079, 2080, 656, 1151, 651, 166, 167, 748, 1705, 179, - /* 100 */ 14, 2164, 1801, 351, 1958, 378, 2160, 1491, 38, 37, + /* 0 */ 1971, 1842, 387, 2072, 429, 604, 627, 2058, 141, 2231, + /* 10 */ 161, 2131, 45, 43, 1614, 1969, 628, 363, 2054, 1855, + /* 20 */ 384, 2058, 1463, 616, 603, 182, 1903, 2058, 496, 2232, + /* 30 */ 605, 2026, 2054, 1544, 492, 1461, 2090, 640, 2054, 44, + /* 40 */ 42, 41, 40, 39, 619, 1168, 332, 1167, 640, 2040, + /* 50 */ 1488, 657, 2050, 2056, 365, 506, 139, 1964, 1539, 1905, + /* 60 */ 222, 2236, 1905, 651, 18, 2231, 2050, 2056, 366, 349, + /* 70 */ 1688, 1469, 2050, 2056, 379, 1904, 1169, 651, 1903, 404, + /* 80 */ 2071, 2235, 176, 651, 2107, 2232, 2234, 109, 2073, 661, + /* 90 */ 2075, 2076, 656, 1149, 651, 166, 1950, 746, 65, 179, + /* 100 */ 14, 2160, 1798, 351, 1954, 378, 2156, 190, 38, 37, /* 110 */ 45, 43, 44, 42, 41, 40, 39, 48, 384, 184, - /* 120 */ 1466, 269, 2172, 615, 176, 133, 614, 2190, 48, 2235, - /* 130 */ 61, 1547, 1153, 1464, 1156, 1157, 1549, 1550, 397, 604, - /* 140 */ 622, 31, 396, 2235, 603, 182, 1959, 38, 37, 2236, - /* 150 */ 605, 44, 42, 41, 40, 39, 1542, 389, 603, 182, - /* 160 */ 1902, 1904, 18, 2236, 605, 1492, 1522, 1532, 1716, 1472, - /* 170 */ 1759, 122, 1548, 1551, 121, 120, 119, 118, 117, 116, - /* 180 */ 115, 114, 113, 579, 1975, 579, 1467, 2235, 1465, 2235, - /* 190 */ 490, 1493, 491, 1729, 65, 748, 375, 1239, 14, 1972, - /* 200 */ 628, 105, 2241, 182, 2241, 182, 640, 2236, 605, 2236, - /* 210 */ 605, 102, 1470, 1471, 2044, 1521, 1524, 1525, 1526, 1527, - /* 220 */ 1528, 1529, 1530, 1531, 653, 649, 1540, 1541, 1543, 1544, - /* 230 */ 1545, 1546, 2, 1241, 1549, 1550, 725, 724, 723, 722, - /* 240 */ 394, 626, 721, 720, 143, 715, 714, 713, 712, 711, - /* 250 */ 710, 709, 156, 705, 704, 703, 393, 392, 700, 699, - /* 260 */ 698, 697, 696, 2240, 1522, 1532, 1715, 2235, 38, 37, - /* 270 */ 1548, 1551, 44, 42, 41, 40, 39, 185, 1686, 185, - /* 280 */ 1494, 1330, 1331, 2239, 1467, 693, 1465, 2236, 2237, 508, - /* 290 */ 61, 1847, 489, 38, 37, 494, 1735, 44, 42, 41, - /* 300 */ 40, 39, 2058, 154, 153, 690, 689, 688, 151, 599, - /* 310 */ 1470, 1471, 2044, 1521, 1524, 1525, 1526, 1527, 1528, 1529, - /* 320 */ 1530, 1531, 653, 649, 1540, 1541, 1543, 1544, 1545, 1546, - /* 330 */ 2, 11, 45, 43, 1762, 1491, 2054, 2060, 1909, 271, - /* 340 */ 384, 1296, 1466, 1492, 695, 362, 1170, 651, 1169, 344, - /* 350 */ 594, 106, 194, 1547, 1907, 1464, 100, 1287, 683, 682, - /* 360 */ 681, 1291, 680, 1293, 1294, 679, 676, 140, 1302, 673, - /* 370 */ 1304, 1305, 670, 667, 1685, 1848, 33, 1171, 1542, 1833, - /* 380 */ 1849, 2076, 38, 37, 18, 568, 44, 42, 41, 40, - /* 390 */ 39, 1472, 1490, 83, 1576, 1645, 82, 545, 544, 543, - /* 400 */ 545, 544, 543, 1714, 535, 136, 539, 535, 136, 539, - /* 410 */ 538, 35, 290, 538, 2094, 537, 542, 748, 537, 542, - /* 420 */ 14, 536, 658, 499, 536, 491, 1729, 2044, 249, 657, - /* 430 */ 45, 43, 1552, 600, 595, 588, 86, 2179, 384, 185, - /* 440 */ 1466, 1250, 591, 590, 1643, 1644, 1646, 1647, 1648, 2044, - /* 450 */ 1577, 1547, 353, 1464, 1249, 641, 1549, 1550, 2075, 1402, - /* 460 */ 1403, 1851, 2111, 2176, 533, 109, 2077, 661, 2079, 2080, - /* 470 */ 656, 187, 651, 1694, 641, 142, 1542, 149, 2135, 2164, - /* 480 */ 61, 1385, 1386, 378, 2160, 532, 1522, 1532, 1856, 1472, - /* 490 */ 54, 693, 1548, 1551, 122, 1401, 1404, 121, 120, 119, - /* 500 */ 118, 117, 116, 115, 114, 113, 1467, 1856, 1465, 154, - /* 510 */ 153, 690, 689, 688, 151, 748, 476, 239, 46, 467, - /* 520 */ 49, 34, 382, 1571, 1572, 1573, 1574, 1575, 1579, 1580, - /* 530 */ 1581, 1582, 1470, 1471, 1693, 1521, 1524, 1525, 1526, 1527, - /* 540 */ 1528, 1529, 1530, 1531, 653, 649, 1540, 1541, 1543, 1544, - /* 550 */ 1545, 1546, 2, 423, 1549, 1550, 1472, 422, 131, 130, - /* 560 */ 129, 128, 127, 126, 125, 124, 123, 272, 498, 38, - /* 570 */ 37, 494, 1735, 44, 42, 41, 40, 39, 164, 2076, - /* 580 */ 198, 197, 1440, 1441, 1522, 1532, 616, 1859, 38, 37, - /* 590 */ 1548, 1551, 44, 42, 41, 40, 39, 2240, 579, 686, - /* 600 */ 1909, 86, 2235, 466, 1467, 2094, 1465, 372, 643, 61, - /* 610 */ 2136, 92, 2094, 598, 1493, 178, 1907, 2241, 182, 139, - /* 620 */ 619, 1831, 2236, 605, 27, 2044, 1852, 657, 1896, 185, - /* 630 */ 1470, 1471, 416, 1521, 1524, 1525, 1526, 1527, 1528, 1529, - /* 640 */ 1530, 1531, 653, 649, 1540, 1541, 1543, 1544, 1545, 1546, - /* 650 */ 2, 45, 43, 2037, 418, 414, 2075, 641, 597, 384, - /* 660 */ 2111, 1466, 1656, 109, 2077, 661, 2079, 2080, 656, 185, - /* 670 */ 651, 641, 1547, 132, 1464, 179, 641, 2164, 1903, 1904, - /* 680 */ 529, 378, 2160, 618, 180, 2172, 2173, 132, 137, 2177, - /* 690 */ 1856, 550, 427, 2179, 534, 1679, 2076, 1542, 579, 73, - /* 700 */ 1713, 616, 2235, 2191, 1856, 641, 560, 38, 37, 1856, - /* 710 */ 1472, 44, 42, 41, 40, 39, 627, 2241, 182, 2175, - /* 720 */ 236, 428, 2236, 605, 41, 40, 39, 641, 238, 2094, - /* 730 */ 616, 641, 237, 693, 139, 553, 748, 658, 1856, 46, - /* 740 */ 547, 11, 2044, 437, 657, 235, 2044, 452, 81, 45, - /* 750 */ 43, 154, 153, 690, 689, 688, 151, 384, 185, 1466, - /* 760 */ 1856, 376, 271, 139, 1856, 625, 627, 1968, 2063, 164, - /* 770 */ 1547, 2038, 1464, 2075, 707, 1549, 1550, 2111, 1858, 2058, - /* 780 */ 168, 2077, 661, 2079, 2080, 656, 69, 651, 1841, 68, - /* 790 */ 89, 339, 1712, 641, 361, 1542, 561, 1711, 2076, 181, - /* 800 */ 2172, 2173, 387, 137, 2177, 1522, 1532, 1954, 1472, 453, - /* 810 */ 164, 1548, 1551, 2054, 2060, 636, 579, 1968, 190, 1858, - /* 820 */ 2235, 580, 2201, 1939, 651, 1467, 1856, 1465, 183, 2172, - /* 830 */ 2173, 2094, 137, 2177, 748, 2241, 182, 14, 2044, 658, - /* 840 */ 2236, 605, 1621, 2044, 2044, 1710, 657, 1557, 1491, 1709, - /* 850 */ 2179, 1470, 1471, 1491, 1521, 1524, 1525, 1526, 1527, 1528, - /* 860 */ 1529, 1530, 1531, 653, 649, 1540, 1541, 1543, 1544, 1545, - /* 870 */ 1546, 2, 1843, 1549, 1550, 2075, 2174, 559, 1708, 2111, - /* 880 */ 1839, 641, 109, 2077, 661, 2079, 2080, 656, 165, 651, - /* 890 */ 557, 2044, 555, 310, 2139, 2044, 2164, 507, 685, 1909, - /* 900 */ 378, 2160, 1707, 1522, 1532, 1704, 377, 308, 72, 1548, - /* 910 */ 1551, 71, 38, 37, 1856, 1907, 44, 42, 41, 40, - /* 920 */ 39, 281, 282, 1467, 2044, 1465, 280, 356, 390, 205, - /* 930 */ 486, 484, 481, 8, 38, 37, 164, 2239, 44, 42, - /* 940 */ 41, 40, 39, 1703, 2076, 1858, 541, 540, 2044, 1470, - /* 950 */ 1471, 2044, 1521, 1524, 1525, 1526, 1527, 1528, 1529, 1530, - /* 960 */ 1531, 653, 649, 1540, 1541, 1543, 1544, 1545, 1546, 2, - /* 970 */ 61, 335, 1909, 1489, 2076, 719, 717, 2094, 447, 388, - /* 980 */ 460, 641, 1614, 474, 641, 658, 473, 446, 1907, 2044, - /* 990 */ 2044, 1254, 657, 357, 1523, 355, 354, 1853, 531, 1523, - /* 1000 */ 241, 443, 533, 475, 1253, 1633, 445, 2094, 108, 244, - /* 1010 */ 641, 641, 373, 1832, 1856, 658, 11, 1856, 9, 1491, - /* 1020 */ 2044, 2075, 657, 532, 652, 2111, 575, 620, 109, 2077, - /* 1030 */ 661, 2079, 2080, 656, 708, 651, 1818, 1702, 1701, 61, - /* 1040 */ 2255, 1494, 2164, 1856, 1856, 1700, 378, 2160, 80, 79, - /* 1050 */ 426, 2075, 352, 189, 641, 2111, 1494, 2198, 327, 2077, - /* 1060 */ 661, 2079, 2080, 656, 433, 651, 478, 13, 12, 1466, - /* 1070 */ 624, 1699, 333, 1749, 1578, 412, 695, 410, 406, 402, - /* 1080 */ 399, 419, 1464, 2044, 2044, 1698, 1697, 1856, 1696, 2076, - /* 1090 */ 645, 2044, 2136, 471, 163, 546, 465, 464, 463, 462, + /* 120 */ 1463, 269, 2168, 615, 176, 133, 614, 2186, 48, 2231, + /* 130 */ 61, 1544, 1151, 1461, 1154, 1155, 1546, 1547, 397, 604, + /* 140 */ 622, 31, 396, 2231, 603, 182, 1955, 38, 37, 2232, + /* 150 */ 605, 44, 42, 41, 40, 39, 1539, 389, 603, 182, + /* 160 */ 1898, 1900, 18, 2232, 605, 1489, 1519, 1529, 1713, 1469, + /* 170 */ 1756, 122, 1545, 1548, 121, 120, 119, 118, 117, 116, + /* 180 */ 115, 114, 113, 579, 1971, 579, 1464, 2231, 1462, 2231, + /* 190 */ 490, 105, 491, 1726, 61, 746, 375, 164, 14, 1968, + /* 200 */ 628, 102, 2237, 182, 2237, 182, 1856, 2232, 605, 2232, + /* 210 */ 605, 1488, 1467, 1468, 2040, 1518, 1521, 1522, 1523, 1524, + /* 220 */ 1525, 1526, 1527, 1528, 653, 649, 1537, 1538, 1540, 1541, + /* 230 */ 1542, 1543, 2, 1488, 1546, 1547, 723, 722, 721, 720, + /* 240 */ 394, 73, 719, 718, 143, 713, 712, 711, 710, 709, + /* 250 */ 708, 707, 156, 703, 702, 701, 393, 392, 698, 697, + /* 260 */ 696, 695, 694, 2236, 1519, 1529, 1712, 2231, 38, 37, + /* 270 */ 1545, 1548, 44, 42, 41, 40, 39, 185, 1683, 185, + /* 280 */ 1491, 1327, 1328, 2235, 1464, 1618, 1462, 2232, 2233, 467, + /* 290 */ 81, 1488, 594, 38, 37, 627, 627, 44, 42, 41, + /* 300 */ 40, 39, 691, 154, 153, 688, 687, 686, 151, 2175, + /* 310 */ 1467, 1468, 2040, 1518, 1521, 1522, 1523, 1524, 1525, 1526, + /* 320 */ 1527, 1528, 653, 649, 1537, 1538, 1540, 1541, 1542, 1543, + /* 330 */ 2, 11, 45, 43, 1759, 2172, 499, 1844, 491, 1726, + /* 340 */ 384, 1293, 1463, 185, 625, 636, 1964, 1964, 2054, 344, + /* 350 */ 198, 197, 167, 1544, 1702, 1461, 1284, 683, 682, 681, + /* 360 */ 1288, 680, 1290, 1291, 679, 676, 2175, 1299, 673, 1301, + /* 370 */ 1302, 670, 667, 466, 1682, 600, 595, 588, 1539, 1382, + /* 380 */ 1383, 2072, 2050, 2056, 18, 489, 508, 86, 494, 1732, + /* 390 */ 2033, 1469, 2171, 651, 1573, 1642, 640, 545, 544, 543, + /* 400 */ 545, 544, 543, 353, 535, 136, 539, 535, 136, 539, + /* 410 */ 538, 416, 1848, 538, 2090, 537, 542, 746, 537, 542, + /* 420 */ 14, 536, 658, 1490, 536, 106, 626, 2040, 249, 657, + /* 430 */ 45, 43, 1549, 418, 414, 579, 1711, 1520, 384, 2231, + /* 440 */ 1463, 140, 591, 590, 1640, 1641, 1643, 1644, 1645, 1845, + /* 450 */ 1574, 1544, 61, 1461, 2237, 182, 1546, 1547, 2071, 2232, + /* 460 */ 605, 100, 2107, 2090, 599, 109, 2073, 661, 2075, 2076, + /* 470 */ 656, 598, 651, 1691, 1905, 142, 1539, 149, 2131, 2160, + /* 480 */ 641, 362, 2040, 378, 2156, 1846, 1519, 1529, 693, 1469, + /* 490 */ 1903, 2175, 1545, 1548, 122, 1487, 187, 121, 120, 119, + /* 500 */ 118, 117, 116, 115, 114, 113, 1464, 498, 1462, 49, + /* 510 */ 494, 1732, 568, 1853, 1935, 746, 597, 2170, 46, 11, + /* 520 */ 1237, 34, 382, 1568, 1569, 1570, 1571, 1572, 1576, 1577, + /* 530 */ 1578, 1579, 1467, 1468, 1690, 1518, 1521, 1522, 1523, 1524, + /* 540 */ 1525, 1526, 1527, 1528, 653, 649, 1537, 1538, 1540, 1541, + /* 550 */ 1542, 1543, 2, 423, 1546, 1547, 1239, 422, 131, 130, + /* 560 */ 129, 128, 127, 126, 125, 124, 123, 641, 178, 38, + /* 570 */ 37, 271, 1554, 44, 42, 41, 40, 39, 1488, 2072, + /* 580 */ 1168, 1892, 1167, 54, 1519, 1529, 616, 1469, 38, 37, + /* 590 */ 1545, 1548, 44, 42, 41, 40, 39, 376, 579, 387, + /* 600 */ 1853, 185, 2231, 1491, 1464, 164, 1462, 164, 41, 40, + /* 610 */ 39, 1169, 2090, 559, 1855, 684, 1855, 2237, 182, 139, + /* 620 */ 619, 1491, 2232, 605, 27, 2040, 557, 657, 555, 1838, + /* 630 */ 1467, 1468, 1830, 1518, 1521, 1522, 1523, 1524, 1525, 1526, + /* 640 */ 1527, 1528, 653, 649, 1537, 1538, 1540, 1541, 1542, 1543, + /* 650 */ 2, 45, 43, 2034, 35, 290, 2071, 272, 185, 384, + /* 660 */ 2107, 1463, 1653, 109, 2073, 661, 2075, 2076, 656, 447, + /* 670 */ 651, 641, 1544, 1710, 1461, 179, 533, 2160, 446, 1248, + /* 680 */ 532, 378, 2156, 618, 180, 2168, 2169, 427, 137, 2173, + /* 690 */ 238, 550, 1247, 641, 237, 705, 2072, 1539, 579, 61, + /* 700 */ 1709, 92, 2231, 2187, 1853, 641, 560, 38, 37, 132, + /* 710 */ 1469, 44, 42, 41, 40, 39, 529, 2237, 182, 2040, + /* 720 */ 236, 132, 2232, 605, 1520, 1905, 1853, 641, 534, 2090, + /* 730 */ 541, 540, 372, 1899, 1900, 553, 746, 658, 1853, 46, + /* 740 */ 547, 1903, 2040, 241, 657, 235, 2040, 194, 1708, 45, + /* 750 */ 43, 191, 89, 339, 476, 1840, 361, 384, 561, 1463, + /* 760 */ 1853, 691, 154, 153, 688, 687, 686, 151, 2059, 248, + /* 770 */ 1544, 566, 1461, 2071, 1489, 1546, 1547, 2107, 8, 2054, + /* 780 */ 168, 2073, 661, 2075, 2076, 656, 69, 651, 83, 68, + /* 790 */ 33, 82, 13, 12, 2040, 1539, 38, 37, 2072, 641, + /* 800 */ 44, 42, 41, 40, 39, 1519, 1529, 1707, 1469, 1829, + /* 810 */ 1836, 1545, 1548, 2050, 2056, 428, 579, 641, 281, 282, + /* 820 */ 2231, 580, 2197, 280, 651, 1464, 2236, 1462, 1154, 1155, + /* 830 */ 390, 2090, 1853, 437, 746, 2237, 182, 14, 164, 658, + /* 840 */ 2232, 605, 685, 1490, 2040, 1896, 657, 1855, 185, 1611, + /* 850 */ 1853, 1467, 1468, 2040, 1518, 1521, 1522, 1523, 1524, 1525, + /* 860 */ 1526, 1527, 1528, 653, 649, 1537, 1538, 1540, 1541, 1542, + /* 870 */ 1543, 2, 693, 1546, 1547, 2071, 1706, 717, 715, 2107, + /* 880 */ 239, 641, 109, 2073, 661, 2075, 2076, 656, 165, 651, + /* 890 */ 1399, 1400, 1587, 310, 2135, 616, 2160, 452, 1733, 1905, + /* 900 */ 378, 2156, 1705, 1519, 1529, 1704, 377, 308, 72, 1545, + /* 910 */ 1548, 71, 38, 37, 1853, 1903, 44, 42, 41, 40, + /* 920 */ 39, 574, 2040, 1464, 1676, 1462, 1398, 1401, 139, 205, + /* 930 */ 486, 484, 481, 1575, 38, 37, 1701, 2235, 44, 42, + /* 940 */ 41, 40, 39, 1700, 2072, 1437, 1438, 741, 2040, 1467, + /* 950 */ 1468, 2040, 1518, 1521, 1522, 1523, 1524, 1525, 1526, 1527, + /* 960 */ 1528, 653, 649, 1537, 1538, 1540, 1541, 1542, 1543, 2, + /* 970 */ 61, 335, 1905, 1486, 2072, 2180, 1607, 2090, 1699, 388, + /* 980 */ 460, 1857, 2040, 474, 641, 658, 473, 1607, 1903, 2040, + /* 990 */ 2040, 271, 657, 181, 2168, 2169, 1252, 137, 2173, 1698, + /* 1000 */ 453, 443, 244, 475, 32, 1630, 445, 2090, 108, 1251, + /* 1010 */ 641, 641, 373, 1799, 1580, 658, 1828, 1853, 1685, 1686, + /* 1020 */ 2040, 2071, 657, 641, 2040, 2107, 507, 1850, 109, 2073, + /* 1030 */ 661, 2075, 2076, 656, 11, 651, 9, 1697, 1696, 575, + /* 1040 */ 2251, 61, 2160, 1853, 1853, 2040, 378, 2156, 80, 79, + /* 1050 */ 426, 2071, 352, 189, 641, 2107, 1853, 2194, 327, 2073, + /* 1060 */ 661, 2075, 2076, 656, 433, 651, 643, 1695, 2132, 1463, + /* 1070 */ 620, 478, 333, 611, 645, 412, 2132, 410, 406, 402, + /* 1080 */ 399, 419, 1461, 2040, 2040, 1488, 1694, 1853, 1693, 2072, + /* 1090 */ 652, 578, 86, 471, 163, 1610, 465, 464, 463, 462, /* 1100 */ 459, 458, 457, 456, 455, 451, 450, 449, 448, 334, - /* 1110 */ 440, 439, 438, 641, 435, 434, 350, 2044, 1472, 185, - /* 1120 */ 641, 608, 2094, 641, 421, 641, 420, 641, 574, 285, - /* 1130 */ 658, 2044, 2044, 152, 2044, 2044, 638, 657, 2076, 639, - /* 1140 */ 1954, 291, 1610, 391, 748, 32, 1856, 2012, 1954, 304, - /* 1150 */ 419, 192, 1886, 1856, 687, 1583, 1856, 1900, 1856, 196, - /* 1160 */ 1856, 1156, 1157, 2184, 1610, 1523, 2075, 1590, 1834, 691, - /* 1170 */ 2111, 2094, 1900, 109, 2077, 661, 2079, 2080, 656, 658, - /* 1180 */ 651, 51, 611, 3, 2044, 2255, 657, 2164, 185, 191, - /* 1190 */ 53, 378, 2160, 240, 430, 692, 1742, 2076, 1900, 145, - /* 1200 */ 152, 134, 2211, 607, 248, 228, 230, 431, 226, 229, - /* 1210 */ 232, 234, 147, 231, 233, 2075, 1688, 1689, 548, 2111, - /* 1220 */ 2076, 247, 109, 2077, 661, 2079, 2080, 656, 1613, 651, - /* 1230 */ 2094, 1802, 152, 1467, 2255, 1465, 2164, 563, 658, 562, - /* 1240 */ 378, 2160, 1740, 2044, 1475, 657, 63, 578, 13, 12, - /* 1250 */ 1706, 586, 2204, 2094, 648, 63, 253, 1435, 701, 1470, - /* 1260 */ 1471, 658, 266, 152, 551, 90, 2044, 592, 657, 47, - /* 1270 */ 278, 70, 150, 152, 2075, 63, 1474, 47, 2111, 47, - /* 1280 */ 1219, 109, 2077, 661, 2079, 2080, 656, 2065, 651, 1438, - /* 1290 */ 665, 221, 150, 2255, 152, 2164, 135, 2075, 2076, 378, - /* 1300 */ 2160, 2111, 150, 1642, 109, 2077, 661, 2079, 2080, 656, - /* 1310 */ 2229, 651, 1641, 255, 381, 380, 2255, 395, 2164, 52, - /* 1320 */ 623, 1736, 378, 2160, 1480, 1200, 1399, 283, 633, 287, - /* 1330 */ 1280, 2094, 1584, 2183, 1533, 1547, 303, 1473, 260, 658, - /* 1340 */ 2067, 702, 1963, 223, 2044, 2095, 657, 1308, 609, 1312, - /* 1350 */ 1730, 1319, 2194, 1317, 617, 1897, 265, 2076, 171, 155, - /* 1360 */ 1542, 1201, 268, 1217, 525, 521, 517, 513, 220, 1, - /* 1370 */ 743, 4, 398, 1472, 403, 2075, 1422, 348, 298, 2111, - /* 1380 */ 2076, 195, 109, 2077, 661, 2079, 2080, 656, 432, 651, - /* 1390 */ 2094, 1494, 1964, 436, 2137, 469, 2164, 1478, 658, 647, - /* 1400 */ 378, 2160, 1568, 2044, 441, 657, 1489, 454, 1956, 87, - /* 1410 */ 468, 612, 218, 2094, 461, 470, 479, 477, 200, 480, - /* 1420 */ 199, 658, 483, 482, 202, 485, 2044, 566, 657, 1477, - /* 1430 */ 487, 1495, 488, 497, 2075, 1497, 1492, 500, 2111, 208, - /* 1440 */ 501, 109, 2077, 661, 2079, 2080, 656, 1496, 651, 210, - /* 1450 */ 502, 1498, 503, 644, 213, 2164, 215, 2075, 505, 378, - /* 1460 */ 2160, 2111, 2076, 509, 110, 2077, 661, 2079, 2080, 656, - /* 1470 */ 84, 651, 579, 85, 219, 1173, 2235, 526, 2164, 217, - /* 1480 */ 211, 528, 2163, 2160, 216, 527, 504, 530, 1481, 111, - /* 1490 */ 1476, 2241, 182, 338, 1846, 2094, 2236, 605, 225, 2021, - /* 1500 */ 1842, 227, 209, 658, 157, 158, 1844, 565, 2044, 1840, - /* 1510 */ 657, 2076, 567, 159, 1484, 1486, 160, 88, 148, 242, - /* 1520 */ 299, 2018, 569, 2017, 245, 576, 573, 649, 1540, 1541, - /* 1530 */ 1543, 1544, 1545, 1546, 2076, 570, 583, 593, 2210, 2075, - /* 1540 */ 2195, 2205, 631, 2111, 2094, 602, 110, 2077, 661, 2079, - /* 1550 */ 2080, 656, 658, 651, 251, 589, 254, 2044, 7, 657, - /* 1560 */ 2164, 367, 596, 2209, 646, 2160, 172, 2094, 2186, 584, - /* 1570 */ 582, 261, 259, 2258, 263, 655, 262, 581, 1610, 264, - /* 1580 */ 2044, 613, 657, 368, 610, 138, 1493, 2180, 659, 371, - /* 1590 */ 621, 1499, 2111, 300, 1969, 110, 2077, 661, 2079, 2080, - /* 1600 */ 656, 629, 651, 273, 95, 634, 2076, 301, 97, 2164, - /* 1610 */ 635, 2075, 99, 343, 2160, 2111, 630, 60, 326, 2077, - /* 1620 */ 661, 2079, 2080, 656, 654, 651, 642, 2129, 267, 2076, - /* 1630 */ 2234, 1983, 1982, 1981, 374, 1857, 302, 2145, 101, 2094, - /* 1640 */ 663, 1901, 1819, 751, 305, 744, 745, 658, 747, 314, - /* 1650 */ 294, 328, 2044, 318, 657, 307, 309, 297, 50, 340, - /* 1660 */ 341, 2036, 2094, 2076, 2035, 2034, 329, 77, 2031, 400, - /* 1670 */ 658, 401, 175, 1457, 1458, 2044, 188, 657, 741, 737, - /* 1680 */ 733, 729, 295, 2075, 405, 2029, 407, 2111, 408, 2076, - /* 1690 */ 169, 2077, 661, 2079, 2080, 656, 2094, 651, 409, 2028, - /* 1700 */ 411, 2027, 413, 2026, 658, 415, 2075, 2025, 417, 2044, - /* 1710 */ 2111, 657, 78, 110, 2077, 661, 2079, 2080, 656, 1425, - /* 1720 */ 651, 1424, 2094, 107, 1995, 1994, 288, 2164, 1993, 424, - /* 1730 */ 658, 425, 2161, 1992, 1991, 2044, 1376, 657, 1947, 1946, - /* 1740 */ 2075, 1944, 606, 2256, 2111, 1943, 144, 168, 2077, 661, - /* 1750 */ 2079, 2080, 656, 2076, 651, 1942, 1945, 1941, 637, 193, - /* 1760 */ 1935, 442, 444, 1949, 1934, 1933, 2075, 1940, 1938, 1937, - /* 1770 */ 2111, 1936, 1932, 320, 2077, 661, 2079, 2080, 656, 1931, - /* 1780 */ 651, 1930, 1929, 1928, 1927, 2076, 2094, 1926, 1925, 2202, - /* 1790 */ 1924, 1923, 1922, 275, 658, 1921, 1920, 1919, 274, 2044, - /* 1800 */ 146, 657, 1918, 1917, 1948, 1916, 1915, 1378, 1914, 1913, - /* 1810 */ 2076, 1912, 1911, 472, 1429, 1910, 243, 601, 2094, 336, - /* 1820 */ 1251, 1255, 1765, 337, 201, 1764, 655, 1763, 1247, 1761, - /* 1830 */ 2075, 2044, 203, 657, 2111, 204, 1725, 169, 2077, 661, - /* 1840 */ 2079, 2080, 656, 2094, 651, 177, 1724, 2064, 383, 206, - /* 1850 */ 2008, 658, 75, 1159, 1158, 76, 2044, 493, 657, 2076, - /* 1860 */ 207, 495, 2075, 2002, 1990, 212, 2111, 214, 1989, 326, - /* 1870 */ 2077, 661, 2079, 2080, 656, 2076, 651, 1967, 2130, 1835, - /* 1880 */ 1760, 1758, 1193, 510, 512, 511, 1756, 2075, 514, 515, - /* 1890 */ 2257, 2111, 2094, 516, 327, 2077, 661, 2079, 2080, 656, - /* 1900 */ 658, 651, 1754, 519, 518, 2044, 520, 657, 2094, 1752, - /* 1910 */ 522, 524, 1739, 385, 523, 1738, 658, 1721, 1837, 1324, - /* 1920 */ 1836, 2044, 1323, 657, 2076, 1238, 224, 62, 1237, 1236, - /* 1930 */ 1235, 1234, 716, 718, 1231, 1229, 564, 1750, 1230, 1228, - /* 1940 */ 2111, 2076, 1743, 322, 2077, 661, 2079, 2080, 656, 358, - /* 1950 */ 651, 359, 2075, 1741, 360, 549, 2111, 2094, 2076, 327, - /* 1960 */ 2077, 661, 2079, 2080, 656, 658, 651, 1720, 552, 554, - /* 1970 */ 2044, 1719, 657, 556, 2094, 1718, 558, 112, 1445, 1447, - /* 1980 */ 1444, 2007, 658, 1449, 1431, 26, 2001, 2044, 571, 657, - /* 1990 */ 1988, 2094, 55, 246, 1986, 2240, 577, 66, 572, 658, - /* 2000 */ 19, 2075, 364, 16, 2044, 2111, 657, 2076, 311, 2077, - /* 2010 */ 661, 2079, 2080, 656, 58, 651, 28, 162, 2075, 5, - /* 2020 */ 1658, 250, 2111, 2076, 585, 312, 2077, 661, 2079, 2080, - /* 2030 */ 656, 6, 651, 59, 587, 2075, 64, 252, 258, 2111, - /* 2040 */ 2094, 2076, 313, 2077, 661, 2079, 2080, 656, 658, 651, - /* 2050 */ 257, 1640, 170, 2044, 2065, 657, 2094, 30, 256, 29, - /* 2060 */ 21, 1632, 91, 1673, 658, 1678, 1679, 1672, 270, 2044, - /* 2070 */ 369, 657, 1677, 1676, 2094, 370, 173, 1607, 1606, 1987, - /* 2080 */ 57, 1985, 658, 93, 2075, 56, 20, 2044, 2111, 657, - /* 2090 */ 2076, 319, 2077, 661, 2079, 2080, 656, 17, 651, 1984, - /* 2100 */ 2075, 1966, 94, 276, 2111, 22, 2076, 323, 2077, 661, - /* 2110 */ 2079, 2080, 656, 1965, 651, 277, 1638, 279, 2075, 284, - /* 2120 */ 67, 96, 2111, 2094, 286, 315, 2077, 661, 2079, 2080, - /* 2130 */ 656, 658, 651, 102, 98, 289, 2044, 23, 657, 2094, - /* 2140 */ 632, 1559, 10, 1558, 12, 1482, 2114, 658, 1537, 650, - /* 2150 */ 174, 1535, 2044, 1569, 657, 36, 186, 1514, 1534, 15, - /* 2160 */ 24, 660, 1506, 2076, 25, 662, 1309, 2075, 664, 386, - /* 2170 */ 666, 2111, 1306, 668, 324, 2077, 661, 2079, 2080, 656, - /* 2180 */ 2076, 651, 671, 2075, 669, 674, 1303, 2111, 1297, 672, - /* 2190 */ 316, 2077, 661, 2079, 2080, 656, 2094, 651, 675, 677, - /* 2200 */ 1301, 1295, 678, 1300, 658, 1286, 684, 103, 104, 2044, - /* 2210 */ 292, 657, 1318, 2094, 1299, 1298, 74, 1314, 1191, 694, - /* 2220 */ 1225, 658, 1224, 1223, 1222, 1221, 2044, 1220, 657, 2076, - /* 2230 */ 1245, 1218, 706, 1216, 1215, 1214, 1212, 293, 1211, 1210, - /* 2240 */ 2075, 1209, 1208, 1207, 2111, 2076, 1206, 325, 2077, 661, - /* 2250 */ 2079, 2080, 656, 1242, 651, 1240, 1203, 2075, 1202, 1199, - /* 2260 */ 1198, 2111, 2094, 1197, 317, 2077, 661, 2079, 2080, 656, - /* 2270 */ 658, 651, 1196, 1757, 726, 2044, 727, 657, 2094, 728, - /* 2280 */ 1755, 731, 730, 732, 1753, 734, 658, 736, 1751, 738, - /* 2290 */ 735, 2044, 740, 657, 2076, 1737, 742, 739, 1148, 1717, - /* 2300 */ 296, 1468, 746, 750, 306, 1692, 2075, 749, 1692, 1692, - /* 2310 */ 2111, 1692, 1692, 330, 2077, 661, 2079, 2080, 656, 2076, - /* 2320 */ 651, 1692, 2075, 1692, 1692, 1692, 2111, 2094, 1692, 331, - /* 2330 */ 2077, 661, 2079, 2080, 656, 658, 651, 1692, 1692, 1692, - /* 2340 */ 2044, 1692, 657, 2076, 1692, 1692, 1692, 1692, 1692, 1692, - /* 2350 */ 1692, 1692, 2094, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 2360 */ 658, 1692, 1692, 1692, 1692, 2044, 1692, 657, 1692, 1692, - /* 2370 */ 1692, 2075, 1692, 1692, 1692, 2111, 2094, 1692, 2088, 2077, - /* 2380 */ 661, 2079, 2080, 656, 658, 651, 1692, 1692, 1692, 2044, - /* 2390 */ 1692, 657, 1692, 1692, 1692, 1692, 2075, 1692, 1692, 1692, - /* 2400 */ 2111, 1692, 1692, 2087, 2077, 661, 2079, 2080, 656, 1692, - /* 2410 */ 651, 1692, 1692, 1692, 2076, 1692, 1692, 1692, 1692, 1692, - /* 2420 */ 2075, 1692, 1692, 1692, 2111, 1692, 1692, 2086, 2077, 661, - /* 2430 */ 2079, 2080, 656, 1692, 651, 1692, 2076, 1692, 1692, 1692, - /* 2440 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 2094, 1692, 1692, - /* 2450 */ 1692, 1692, 1692, 1692, 1692, 658, 1692, 1692, 1692, 1692, - /* 2460 */ 2044, 1692, 657, 2076, 1692, 1692, 1692, 1692, 1692, 2094, - /* 2470 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 658, 1692, 1692, - /* 2480 */ 1692, 1692, 2044, 1692, 657, 1692, 1692, 1692, 1692, 1692, - /* 2490 */ 1692, 2075, 1692, 1692, 1692, 2111, 2094, 1692, 345, 2077, - /* 2500 */ 661, 2079, 2080, 656, 658, 651, 1692, 1692, 1692, 2044, - /* 2510 */ 1692, 657, 2076, 2075, 1692, 1692, 1692, 2111, 1692, 1692, - /* 2520 */ 346, 2077, 661, 2079, 2080, 656, 1692, 651, 2076, 1692, - /* 2530 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 2540 */ 2075, 1692, 1692, 1692, 2111, 2094, 1692, 342, 2077, 661, - /* 2550 */ 2079, 2080, 656, 658, 651, 1692, 1692, 1692, 2044, 1692, - /* 2560 */ 657, 2094, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 658, - /* 2570 */ 1692, 1692, 1692, 1692, 2044, 1692, 657, 2076, 1692, 1692, - /* 2580 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 2075, - /* 2590 */ 1692, 1692, 1692, 2111, 1692, 1692, 347, 2077, 661, 2079, - /* 2600 */ 2080, 656, 1692, 651, 1692, 659, 1692, 1692, 1692, 2111, - /* 2610 */ 2094, 1692, 322, 2077, 661, 2079, 2080, 656, 658, 651, - /* 2620 */ 1692, 1692, 1692, 2044, 1692, 657, 1692, 1692, 1692, 1692, - /* 2630 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 2640 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 2650 */ 1692, 1692, 1692, 1692, 2075, 1692, 1692, 1692, 2111, 1692, - /* 2660 */ 1692, 321, 2077, 661, 2079, 2080, 656, 1692, 651, + /* 1110 */ 440, 439, 438, 2040, 435, 434, 350, 1849, 1469, 185, + /* 1120 */ 641, 608, 2090, 641, 641, 641, 421, 706, 420, 1815, + /* 1130 */ 658, 1950, 2040, 2200, 2040, 2040, 624, 657, 2072, 285, + /* 1140 */ 638, 639, 192, 1950, 746, 691, 154, 153, 688, 687, + /* 1150 */ 686, 151, 419, 1853, 196, 2008, 1853, 1853, 1853, 304, + /* 1160 */ 152, 689, 1882, 52, 1896, 641, 2071, 641, 1831, 430, + /* 1170 */ 2107, 2090, 1746, 109, 2073, 661, 2075, 2076, 656, 658, + /* 1180 */ 651, 291, 431, 391, 2040, 2251, 657, 2160, 13, 12, + /* 1190 */ 185, 378, 2156, 690, 546, 152, 1896, 2072, 1853, 152, + /* 1200 */ 1853, 240, 2207, 607, 51, 145, 3, 134, 228, 1472, + /* 1210 */ 230, 226, 147, 229, 247, 2071, 1471, 53, 232, 2107, + /* 1220 */ 2072, 231, 109, 2073, 661, 2075, 2076, 656, 1739, 651, + /* 1230 */ 2090, 1520, 1737, 1464, 2251, 1462, 2160, 63, 658, 234, + /* 1240 */ 378, 2156, 233, 2040, 563, 657, 562, 648, 1703, 266, + /* 1250 */ 548, 586, 1432, 2090, 551, 63, 1435, 592, 90, 1467, + /* 1260 */ 1468, 658, 253, 152, 47, 221, 2040, 699, 657, 278, + /* 1270 */ 70, 150, 260, 152, 2071, 63, 2091, 47, 2107, 47, + /* 1280 */ 395, 109, 2073, 661, 2075, 2076, 656, 2061, 651, 1217, + /* 1290 */ 665, 1893, 150, 2251, 1639, 2160, 152, 2071, 2072, 378, + /* 1300 */ 2156, 2107, 612, 700, 109, 2073, 661, 2075, 2076, 656, + /* 1310 */ 2225, 651, 1638, 135, 381, 380, 2251, 616, 2160, 255, + /* 1320 */ 623, 1396, 378, 2156, 1477, 1215, 283, 633, 287, 1959, + /* 1330 */ 1278, 2090, 1581, 2179, 1530, 1544, 303, 1470, 1198, 658, + /* 1340 */ 2063, 1727, 2190, 223, 2040, 150, 657, 1305, 609, 1309, + /* 1350 */ 139, 617, 268, 1316, 265, 1, 4, 2072, 171, 398, + /* 1360 */ 1539, 403, 1475, 348, 525, 521, 517, 513, 220, 1474, + /* 1370 */ 1314, 298, 1419, 1469, 1199, 2071, 195, 432, 1960, 2107, + /* 1380 */ 2072, 356, 109, 2073, 661, 2075, 2076, 656, 1491, 651, + /* 1390 */ 2090, 436, 441, 1486, 2133, 1565, 2160, 454, 658, 647, + /* 1400 */ 378, 2156, 155, 2040, 469, 657, 1952, 461, 480, 87, + /* 1410 */ 468, 477, 218, 2090, 470, 183, 2168, 2169, 479, 137, + /* 1420 */ 2173, 658, 199, 200, 482, 483, 2040, 202, 657, 485, + /* 1430 */ 487, 1492, 488, 497, 2071, 1494, 500, 208, 2107, 1489, + /* 1440 */ 501, 109, 2073, 661, 2075, 2076, 656, 357, 651, 355, + /* 1450 */ 354, 1493, 531, 644, 210, 2160, 1495, 2071, 502, 378, + /* 1460 */ 2156, 2107, 2072, 503, 110, 2073, 661, 2075, 2076, 656, + /* 1470 */ 213, 651, 505, 533, 215, 84, 509, 532, 2160, 217, + /* 1480 */ 211, 85, 2159, 2156, 216, 219, 504, 1171, 1478, 2017, + /* 1490 */ 1473, 526, 527, 528, 2014, 2090, 2013, 530, 1843, 338, + /* 1500 */ 111, 565, 209, 658, 567, 225, 1839, 227, 2040, 157, + /* 1510 */ 657, 2072, 158, 88, 1481, 1483, 1841, 1837, 159, 242, + /* 1520 */ 148, 160, 569, 299, 570, 573, 2191, 649, 1537, 1538, + /* 1530 */ 1540, 1541, 1542, 1543, 2072, 245, 576, 593, 2206, 2071, + /* 1540 */ 631, 583, 589, 2107, 2090, 2201, 110, 2073, 661, 2075, + /* 1550 */ 2076, 656, 658, 651, 2205, 367, 596, 2040, 7, 657, + /* 1560 */ 2160, 251, 254, 602, 646, 2156, 259, 2090, 368, 581, + /* 1570 */ 267, 582, 172, 584, 261, 655, 262, 2182, 2254, 613, + /* 1580 */ 2040, 264, 657, 1607, 610, 138, 263, 1490, 659, 2176, + /* 1590 */ 2230, 371, 2107, 273, 621, 110, 2073, 661, 2075, 2076, + /* 1600 */ 656, 95, 651, 1496, 1965, 97, 2072, 300, 301, 2160, + /* 1610 */ 634, 2071, 635, 343, 2156, 2107, 99, 1854, 326, 2073, + /* 1620 */ 661, 2075, 2076, 656, 654, 651, 642, 2125, 302, 2072, + /* 1630 */ 629, 60, 2141, 630, 1979, 101, 1978, 1977, 374, 2090, + /* 1640 */ 663, 1897, 1816, 749, 742, 305, 743, 658, 745, 314, + /* 1650 */ 2032, 294, 2040, 329, 657, 328, 340, 297, 309, 341, + /* 1660 */ 2031, 2030, 2090, 2072, 50, 307, 2027, 318, 77, 400, + /* 1670 */ 658, 401, 175, 1454, 1455, 2040, 188, 657, 739, 735, + /* 1680 */ 731, 727, 295, 2071, 405, 2025, 407, 2107, 408, 2072, + /* 1690 */ 169, 2073, 661, 2075, 2076, 656, 2090, 651, 409, 2024, + /* 1700 */ 411, 2023, 413, 2022, 658, 415, 2071, 2021, 417, 2040, + /* 1710 */ 2107, 657, 78, 110, 2073, 661, 2075, 2076, 656, 1991, + /* 1720 */ 651, 1422, 2090, 107, 1421, 1990, 288, 2160, 1989, 424, + /* 1730 */ 658, 425, 2157, 1988, 1987, 2040, 1373, 657, 1943, 1942, + /* 1740 */ 2071, 1940, 606, 2252, 2107, 1939, 144, 168, 2073, 661, + /* 1750 */ 2075, 2076, 656, 2072, 651, 1938, 1941, 1937, 637, 193, + /* 1760 */ 1931, 442, 444, 1945, 1930, 1929, 2071, 1936, 1934, 1933, + /* 1770 */ 2107, 1932, 1928, 320, 2073, 661, 2075, 2076, 656, 1927, + /* 1780 */ 651, 1926, 1925, 1924, 1923, 2072, 2090, 1922, 1921, 2198, + /* 1790 */ 1920, 1919, 1918, 275, 658, 1917, 1916, 1915, 274, 2040, + /* 1800 */ 146, 657, 1914, 1913, 1944, 1912, 1911, 1375, 1910, 1909, + /* 1810 */ 2072, 1908, 1907, 472, 1426, 1906, 243, 601, 2090, 336, + /* 1820 */ 1249, 1253, 1762, 337, 201, 1761, 655, 1760, 1245, 1758, + /* 1830 */ 2071, 2040, 203, 657, 2107, 204, 1722, 169, 2073, 661, + /* 1840 */ 2075, 2076, 656, 2090, 651, 177, 1721, 2060, 383, 206, + /* 1850 */ 2004, 658, 75, 1157, 1156, 76, 2040, 493, 657, 2072, + /* 1860 */ 207, 495, 2071, 1998, 1986, 212, 2107, 214, 1985, 326, + /* 1870 */ 2073, 661, 2075, 2076, 656, 2072, 651, 1963, 2126, 1832, + /* 1880 */ 1757, 1755, 1191, 510, 512, 511, 1753, 2071, 514, 515, + /* 1890 */ 2253, 2107, 2090, 516, 327, 2073, 661, 2075, 2076, 656, + /* 1900 */ 658, 651, 1751, 519, 518, 2040, 520, 657, 2090, 1749, + /* 1910 */ 522, 524, 1736, 385, 523, 1735, 658, 1718, 1834, 1321, + /* 1920 */ 1833, 2040, 1320, 657, 2072, 1236, 224, 62, 1235, 1234, + /* 1930 */ 1233, 1232, 714, 716, 1229, 1227, 564, 1747, 1228, 1226, + /* 1940 */ 2107, 2072, 1740, 322, 2073, 661, 2075, 2076, 656, 358, + /* 1950 */ 651, 359, 2071, 1738, 360, 549, 2107, 2090, 2072, 327, + /* 1960 */ 2073, 661, 2075, 2076, 656, 658, 651, 1717, 552, 554, + /* 1970 */ 2040, 1716, 657, 556, 2090, 1715, 558, 112, 1442, 1444, + /* 1980 */ 1441, 2003, 658, 1446, 1428, 26, 1997, 2040, 571, 657, + /* 1990 */ 1984, 2090, 55, 246, 1982, 2236, 577, 66, 572, 658, + /* 2000 */ 19, 2071, 364, 16, 2040, 2107, 657, 2072, 311, 2073, + /* 2010 */ 661, 2075, 2076, 656, 58, 651, 28, 162, 2071, 5, + /* 2020 */ 1655, 250, 2107, 2072, 585, 312, 2073, 661, 2075, 2076, + /* 2030 */ 656, 6, 651, 59, 587, 2071, 64, 252, 258, 2107, + /* 2040 */ 2090, 2072, 313, 2073, 661, 2075, 2076, 656, 658, 651, + /* 2050 */ 257, 1637, 170, 2040, 2061, 657, 2090, 30, 256, 29, + /* 2060 */ 21, 1629, 91, 1670, 658, 1675, 1676, 1669, 270, 2040, + /* 2070 */ 369, 657, 1674, 1673, 2090, 370, 173, 1604, 1603, 1983, + /* 2080 */ 57, 1981, 658, 93, 2071, 56, 20, 2040, 2107, 657, + /* 2090 */ 2072, 319, 2073, 661, 2075, 2076, 656, 17, 651, 1980, + /* 2100 */ 2071, 1962, 94, 276, 2107, 22, 2072, 323, 2073, 661, + /* 2110 */ 2075, 2076, 656, 1961, 651, 277, 1635, 279, 2071, 284, + /* 2120 */ 67, 96, 2107, 2090, 286, 315, 2073, 661, 2075, 2076, + /* 2130 */ 656, 658, 651, 102, 98, 289, 2040, 23, 657, 2090, + /* 2140 */ 632, 1556, 10, 1555, 12, 1479, 2110, 658, 1534, 650, + /* 2150 */ 174, 1532, 2040, 1566, 657, 36, 186, 1511, 1531, 15, + /* 2160 */ 24, 660, 1503, 2072, 25, 662, 1306, 2071, 664, 386, + /* 2170 */ 666, 2107, 1303, 668, 324, 2073, 661, 2075, 2076, 656, + /* 2180 */ 2072, 651, 671, 2071, 669, 674, 1300, 2107, 1294, 672, + /* 2190 */ 316, 2073, 661, 2075, 2076, 656, 2090, 651, 675, 677, + /* 2200 */ 1298, 1292, 678, 292, 658, 1315, 103, 104, 1297, 2040, + /* 2210 */ 74, 657, 1296, 2090, 1295, 1311, 1189, 692, 1223, 1222, + /* 2220 */ 1221, 658, 1220, 1219, 1218, 1216, 2040, 1214, 657, 2072, + /* 2230 */ 1213, 1212, 1243, 704, 1210, 293, 1209, 1208, 1207, 1206, + /* 2240 */ 2071, 1205, 1204, 1240, 2107, 2072, 1238, 325, 2073, 661, + /* 2250 */ 2075, 2076, 656, 1201, 651, 1200, 1197, 2071, 1196, 1195, + /* 2260 */ 1194, 2107, 2090, 1754, 317, 2073, 661, 2075, 2076, 656, + /* 2270 */ 658, 651, 725, 724, 726, 2040, 1752, 657, 2090, 728, + /* 2280 */ 729, 730, 1750, 732, 733, 1748, 658, 736, 734, 737, + /* 2290 */ 738, 2040, 1734, 657, 2072, 740, 1146, 1714, 748, 296, + /* 2300 */ 744, 1689, 1465, 306, 747, 1689, 2071, 1689, 1689, 1689, + /* 2310 */ 2107, 1689, 1689, 330, 2073, 661, 2075, 2076, 656, 2072, + /* 2320 */ 651, 1689, 2071, 1689, 1689, 1689, 2107, 2090, 1689, 331, + /* 2330 */ 2073, 661, 2075, 2076, 656, 658, 651, 1689, 1689, 1689, + /* 2340 */ 2040, 1689, 657, 2072, 1689, 1689, 1689, 1689, 1689, 1689, + /* 2350 */ 1689, 1689, 2090, 1689, 1689, 1689, 1689, 1689, 1689, 1689, + /* 2360 */ 658, 1689, 1689, 1689, 1689, 2040, 1689, 657, 1689, 1689, + /* 2370 */ 1689, 2071, 1689, 1689, 1689, 2107, 2090, 1689, 2084, 2073, + /* 2380 */ 661, 2075, 2076, 656, 658, 651, 1689, 1689, 1689, 2040, + /* 2390 */ 1689, 657, 1689, 1689, 1689, 1689, 2071, 1689, 1689, 1689, + /* 2400 */ 2107, 1689, 1689, 2083, 2073, 661, 2075, 2076, 656, 1689, + /* 2410 */ 651, 1689, 1689, 1689, 2072, 1689, 1689, 1689, 1689, 1689, + /* 2420 */ 2071, 1689, 1689, 1689, 2107, 1689, 1689, 2082, 2073, 661, + /* 2430 */ 2075, 2076, 656, 1689, 651, 1689, 2072, 1689, 1689, 1689, + /* 2440 */ 1689, 1689, 1689, 1689, 1689, 1689, 1689, 2090, 1689, 1689, + /* 2450 */ 1689, 1689, 1689, 1689, 1689, 658, 1689, 1689, 1689, 1689, + /* 2460 */ 2040, 1689, 657, 2072, 1689, 1689, 1689, 1689, 1689, 2090, + /* 2470 */ 1689, 1689, 1689, 1689, 1689, 1689, 1689, 658, 1689, 1689, + /* 2480 */ 1689, 1689, 2040, 1689, 657, 1689, 1689, 1689, 1689, 1689, + /* 2490 */ 1689, 2071, 1689, 1689, 1689, 2107, 2090, 1689, 345, 2073, + /* 2500 */ 661, 2075, 2076, 656, 658, 651, 1689, 1689, 1689, 2040, + /* 2510 */ 1689, 657, 2072, 2071, 1689, 1689, 1689, 2107, 1689, 1689, + /* 2520 */ 346, 2073, 661, 2075, 2076, 656, 1689, 651, 2072, 1689, + /* 2530 */ 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, + /* 2540 */ 2071, 1689, 1689, 1689, 2107, 2090, 1689, 342, 2073, 661, + /* 2550 */ 2075, 2076, 656, 658, 651, 1689, 1689, 1689, 2040, 1689, + /* 2560 */ 657, 2090, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 658, + /* 2570 */ 1689, 1689, 1689, 1689, 2040, 1689, 657, 2072, 1689, 1689, + /* 2580 */ 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 2071, + /* 2590 */ 1689, 1689, 1689, 2107, 1689, 1689, 347, 2073, 661, 2075, + /* 2600 */ 2076, 656, 1689, 651, 1689, 659, 1689, 1689, 1689, 2107, + /* 2610 */ 2090, 1689, 322, 2073, 661, 2075, 2076, 656, 658, 651, + /* 2620 */ 1689, 1689, 1689, 2040, 1689, 657, 1689, 1689, 1689, 1689, + /* 2630 */ 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, + /* 2640 */ 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, 1689, + /* 2650 */ 1689, 1689, 1689, 1689, 2071, 1689, 1689, 1689, 2107, 1689, + /* 2660 */ 1689, 321, 2073, 661, 2075, 2076, 656, 1689, 651, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 379, 365, 356, 331, 339, 441, 339, 366, 425, 445, @@ -496,8 +496,8 @@ static const YYCODETYPE yy_lookahead[] = { /* 60 */ 35, 441, 364, 422, 64, 445, 411, 412, 413, 371, /* 70 */ 328, 71, 411, 412, 413, 380, 51, 422, 380, 48, /* 80 */ 408, 461, 364, 422, 412, 465, 466, 415, 416, 417, - /* 90 */ 418, 419, 420, 4, 422, 346, 330, 97, 332, 427, - /* 100 */ 100, 429, 353, 385, 386, 433, 434, 20, 8, 9, + /* 90 */ 418, 419, 420, 4, 422, 346, 372, 97, 4, 427, + /* 100 */ 100, 429, 353, 385, 386, 433, 434, 383, 8, 9, /* 110 */ 12, 13, 12, 13, 14, 15, 16, 100, 20, 447, /* 120 */ 22, 437, 438, 439, 364, 441, 442, 455, 100, 445, /* 130 */ 100, 33, 43, 35, 45, 46, 136, 137, 396, 441, @@ -506,160 +506,160 @@ static const YYCODETYPE yy_lookahead[] = { /* 160 */ 378, 379, 64, 465, 466, 20, 166, 167, 331, 71, /* 170 */ 0, 21, 172, 173, 24, 25, 26, 27, 28, 29, /* 180 */ 30, 31, 32, 441, 379, 441, 186, 445, 188, 445, - /* 190 */ 335, 20, 337, 338, 4, 97, 391, 35, 100, 394, - /* 200 */ 395, 100, 460, 461, 460, 461, 20, 465, 466, 465, - /* 210 */ 466, 110, 212, 213, 377, 215, 216, 217, 218, 219, + /* 190 */ 335, 100, 337, 338, 100, 97, 391, 364, 100, 394, + /* 200 */ 395, 110, 460, 461, 460, 461, 373, 465, 466, 465, + /* 210 */ 466, 20, 212, 213, 377, 215, 216, 217, 218, 219, /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - /* 230 */ 230, 231, 232, 71, 136, 137, 66, 67, 68, 69, - /* 240 */ 70, 20, 72, 73, 74, 75, 76, 77, 78, 79, + /* 230 */ 230, 231, 232, 20, 136, 137, 66, 67, 68, 69, + /* 240 */ 70, 111, 72, 73, 74, 75, 76, 77, 78, 79, /* 250 */ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, /* 260 */ 90, 91, 92, 441, 166, 167, 331, 445, 8, 9, /* 270 */ 172, 173, 12, 13, 14, 15, 16, 249, 178, 249, - /* 280 */ 20, 136, 137, 461, 186, 112, 188, 465, 466, 63, - /* 290 */ 100, 366, 336, 8, 9, 339, 340, 12, 13, 14, - /* 300 */ 15, 16, 377, 130, 131, 132, 133, 134, 135, 20, + /* 280 */ 20, 136, 137, 461, 186, 14, 188, 465, 466, 80, + /* 290 */ 160, 20, 171, 8, 9, 339, 339, 12, 13, 14, + /* 300 */ 15, 16, 129, 130, 131, 132, 133, 134, 135, 414, /* 310 */ 212, 213, 377, 215, 216, 217, 218, 219, 220, 221, /* 320 */ 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - /* 330 */ 232, 233, 12, 13, 0, 20, 411, 412, 364, 168, - /* 340 */ 20, 97, 22, 20, 63, 371, 20, 422, 22, 64, - /* 350 */ 171, 343, 58, 33, 380, 35, 343, 113, 114, 115, - /* 360 */ 116, 117, 118, 119, 120, 121, 122, 359, 124, 125, - /* 370 */ 126, 127, 128, 129, 274, 367, 2, 51, 58, 0, - /* 380 */ 367, 331, 8, 9, 64, 111, 12, 13, 14, 15, - /* 390 */ 16, 71, 20, 99, 109, 212, 102, 66, 67, 68, - /* 400 */ 66, 67, 68, 331, 73, 74, 75, 73, 74, 75, - /* 410 */ 79, 430, 431, 79, 364, 84, 85, 97, 84, 85, - /* 420 */ 100, 90, 372, 335, 90, 337, 338, 377, 168, 379, - /* 430 */ 12, 13, 14, 254, 255, 256, 345, 414, 20, 249, - /* 440 */ 22, 22, 259, 260, 261, 262, 263, 264, 265, 377, - /* 450 */ 165, 33, 361, 35, 35, 339, 136, 137, 408, 136, - /* 460 */ 137, 370, 412, 440, 112, 415, 416, 417, 418, 419, - /* 470 */ 420, 355, 422, 0, 339, 425, 58, 427, 428, 429, - /* 480 */ 100, 166, 167, 433, 434, 133, 166, 167, 372, 71, - /* 490 */ 355, 112, 172, 173, 21, 172, 173, 24, 25, 26, - /* 500 */ 27, 28, 29, 30, 31, 32, 186, 372, 188, 130, - /* 510 */ 131, 132, 133, 134, 135, 97, 97, 130, 100, 80, - /* 520 */ 100, 236, 237, 238, 239, 240, 241, 242, 243, 244, + /* 330 */ 232, 233, 12, 13, 0, 440, 335, 366, 337, 338, + /* 340 */ 20, 97, 22, 249, 388, 388, 390, 390, 377, 64, + /* 350 */ 141, 142, 330, 33, 332, 35, 112, 113, 114, 115, + /* 360 */ 116, 117, 118, 119, 120, 121, 414, 123, 124, 125, + /* 370 */ 126, 127, 128, 164, 274, 254, 255, 256, 58, 166, + /* 380 */ 167, 331, 411, 412, 64, 336, 63, 345, 339, 340, + /* 390 */ 396, 71, 440, 422, 109, 212, 20, 66, 67, 68, + /* 400 */ 66, 67, 68, 361, 73, 74, 75, 73, 74, 75, + /* 410 */ 79, 181, 370, 79, 364, 84, 85, 97, 84, 85, + /* 420 */ 100, 90, 372, 20, 90, 343, 20, 377, 168, 379, + /* 430 */ 12, 13, 14, 203, 204, 441, 331, 166, 20, 445, + /* 440 */ 22, 359, 259, 260, 261, 262, 263, 264, 265, 367, + /* 450 */ 165, 33, 100, 35, 460, 461, 136, 137, 408, 465, + /* 460 */ 466, 343, 412, 364, 20, 415, 416, 417, 418, 419, + /* 470 */ 420, 372, 422, 0, 364, 425, 58, 427, 428, 429, + /* 480 */ 339, 371, 377, 433, 434, 367, 166, 167, 63, 71, + /* 490 */ 380, 414, 172, 173, 21, 20, 355, 24, 25, 26, + /* 500 */ 27, 28, 29, 30, 31, 32, 186, 336, 188, 100, + /* 510 */ 339, 340, 111, 372, 0, 97, 417, 440, 100, 233, + /* 520 */ 35, 236, 237, 238, 239, 240, 241, 242, 243, 244, /* 530 */ 245, 246, 212, 213, 0, 215, 216, 217, 218, 219, /* 540 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, /* 550 */ 230, 231, 232, 396, 136, 137, 71, 400, 24, 25, - /* 560 */ 26, 27, 28, 29, 30, 31, 32, 58, 336, 8, - /* 570 */ 9, 339, 340, 12, 13, 14, 15, 16, 364, 331, - /* 580 */ 141, 142, 195, 196, 166, 167, 339, 373, 8, 9, - /* 590 */ 172, 173, 12, 13, 14, 15, 16, 3, 441, 111, - /* 600 */ 364, 345, 445, 164, 186, 364, 188, 371, 426, 100, - /* 610 */ 428, 102, 364, 372, 20, 363, 380, 460, 461, 372, - /* 620 */ 372, 0, 465, 466, 44, 377, 370, 379, 376, 249, - /* 630 */ 212, 213, 181, 215, 216, 217, 218, 219, 220, 221, + /* 560 */ 26, 27, 28, 29, 30, 31, 32, 339, 363, 8, + /* 570 */ 9, 168, 14, 12, 13, 14, 15, 16, 20, 331, + /* 580 */ 20, 376, 22, 355, 166, 167, 339, 71, 8, 9, + /* 590 */ 172, 173, 12, 13, 14, 15, 16, 356, 441, 356, + /* 600 */ 372, 249, 445, 20, 186, 364, 188, 364, 14, 15, + /* 610 */ 16, 51, 364, 21, 373, 111, 373, 460, 461, 372, + /* 620 */ 372, 20, 465, 466, 44, 377, 34, 379, 36, 365, + /* 630 */ 212, 213, 0, 215, 216, 217, 218, 219, 220, 221, /* 640 */ 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - /* 650 */ 232, 12, 13, 396, 203, 204, 408, 339, 417, 20, - /* 660 */ 412, 22, 101, 415, 416, 417, 418, 419, 420, 249, - /* 670 */ 422, 339, 33, 355, 35, 427, 339, 429, 378, 379, - /* 680 */ 362, 433, 434, 436, 437, 438, 439, 355, 441, 442, - /* 690 */ 372, 4, 355, 414, 362, 101, 331, 58, 441, 111, - /* 700 */ 331, 339, 445, 455, 372, 339, 19, 8, 9, 372, - /* 710 */ 71, 12, 13, 14, 15, 16, 339, 460, 461, 440, - /* 720 */ 33, 355, 465, 466, 14, 15, 16, 339, 131, 364, - /* 730 */ 339, 339, 135, 112, 372, 48, 97, 372, 372, 100, - /* 740 */ 53, 233, 377, 355, 379, 58, 377, 355, 160, 12, - /* 750 */ 13, 130, 131, 132, 133, 134, 135, 20, 249, 22, - /* 760 */ 372, 356, 168, 372, 372, 388, 339, 390, 366, 364, - /* 770 */ 33, 396, 35, 408, 71, 136, 137, 412, 373, 377, - /* 780 */ 415, 416, 417, 418, 419, 420, 99, 422, 365, 102, - /* 790 */ 193, 194, 331, 339, 197, 58, 199, 331, 331, 437, - /* 800 */ 438, 439, 356, 441, 442, 166, 167, 372, 71, 355, - /* 810 */ 364, 172, 173, 411, 412, 388, 441, 390, 383, 373, - /* 820 */ 445, 456, 457, 0, 422, 186, 372, 188, 437, 438, - /* 830 */ 439, 364, 441, 442, 97, 460, 461, 100, 377, 372, - /* 840 */ 465, 466, 14, 377, 377, 331, 379, 14, 20, 331, - /* 850 */ 414, 212, 213, 20, 215, 216, 217, 218, 219, 220, + /* 650 */ 232, 12, 13, 396, 430, 431, 408, 58, 249, 20, + /* 660 */ 412, 22, 101, 415, 416, 417, 418, 419, 420, 155, + /* 670 */ 422, 339, 33, 331, 35, 427, 129, 429, 164, 22, + /* 680 */ 133, 433, 434, 436, 437, 438, 439, 355, 441, 442, + /* 690 */ 131, 4, 35, 339, 135, 71, 331, 58, 441, 100, + /* 700 */ 331, 102, 445, 455, 372, 339, 19, 8, 9, 355, + /* 710 */ 71, 12, 13, 14, 15, 16, 362, 460, 461, 377, + /* 720 */ 33, 355, 465, 466, 166, 364, 372, 339, 362, 364, + /* 730 */ 350, 351, 371, 378, 379, 48, 97, 372, 372, 100, + /* 740 */ 53, 380, 377, 355, 379, 58, 377, 58, 331, 12, + /* 750 */ 13, 168, 193, 194, 97, 365, 197, 20, 199, 22, + /* 760 */ 372, 129, 130, 131, 132, 133, 134, 135, 366, 168, + /* 770 */ 33, 396, 35, 408, 20, 136, 137, 412, 39, 377, + /* 780 */ 415, 416, 417, 418, 419, 420, 99, 422, 99, 102, + /* 790 */ 2, 102, 1, 2, 377, 58, 8, 9, 331, 339, + /* 800 */ 12, 13, 14, 15, 16, 166, 167, 331, 71, 0, + /* 810 */ 365, 172, 173, 411, 412, 355, 441, 339, 130, 131, + /* 820 */ 445, 456, 457, 135, 422, 186, 3, 188, 45, 46, + /* 830 */ 356, 364, 372, 355, 97, 460, 461, 100, 364, 372, + /* 840 */ 465, 466, 374, 20, 377, 377, 379, 373, 249, 4, + /* 850 */ 372, 212, 213, 377, 215, 216, 217, 218, 219, 220, /* 860 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - /* 870 */ 231, 232, 365, 136, 137, 408, 440, 21, 331, 412, - /* 880 */ 365, 339, 415, 416, 417, 418, 419, 420, 18, 422, - /* 890 */ 34, 377, 36, 23, 427, 377, 429, 355, 365, 364, + /* 870 */ 231, 232, 63, 136, 137, 408, 331, 350, 351, 412, + /* 880 */ 130, 339, 415, 416, 417, 418, 419, 420, 18, 422, + /* 890 */ 136, 137, 101, 23, 427, 339, 429, 355, 0, 364, /* 900 */ 433, 434, 331, 166, 167, 331, 371, 37, 38, 172, /* 910 */ 173, 41, 8, 9, 372, 380, 12, 13, 14, 15, - /* 920 */ 16, 130, 131, 186, 377, 188, 135, 37, 356, 59, - /* 930 */ 60, 61, 62, 39, 8, 9, 364, 3, 12, 13, - /* 940 */ 14, 15, 16, 331, 331, 373, 350, 351, 377, 212, + /* 920 */ 16, 401, 377, 186, 101, 188, 172, 173, 372, 59, + /* 930 */ 60, 61, 62, 165, 8, 9, 331, 3, 12, 13, + /* 940 */ 14, 15, 16, 331, 331, 195, 196, 49, 377, 212, /* 950 */ 213, 377, 215, 216, 217, 218, 219, 220, 221, 222, /* 960 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, - /* 970 */ 100, 18, 364, 20, 331, 350, 351, 364, 155, 371, - /* 980 */ 27, 339, 4, 30, 339, 372, 33, 164, 380, 377, - /* 990 */ 377, 22, 379, 103, 166, 105, 106, 355, 108, 166, - /* 1000 */ 355, 48, 112, 50, 35, 101, 53, 364, 138, 365, - /* 1010 */ 339, 339, 369, 0, 372, 372, 233, 372, 235, 20, - /* 1020 */ 377, 408, 379, 133, 365, 412, 355, 355, 415, 416, - /* 1030 */ 417, 418, 419, 420, 352, 422, 354, 331, 331, 100, - /* 1040 */ 427, 20, 429, 372, 372, 331, 433, 434, 178, 179, - /* 1050 */ 180, 408, 99, 183, 339, 412, 20, 444, 415, 416, - /* 1060 */ 417, 418, 419, 420, 111, 422, 97, 1, 2, 22, - /* 1070 */ 355, 331, 202, 0, 165, 205, 63, 207, 208, 209, - /* 1080 */ 210, 211, 35, 377, 377, 331, 331, 372, 331, 331, - /* 1090 */ 426, 377, 428, 140, 168, 22, 143, 144, 145, 146, + /* 970 */ 100, 18, 364, 20, 331, 247, 248, 364, 331, 371, + /* 980 */ 27, 365, 377, 30, 339, 372, 33, 248, 380, 377, + /* 990 */ 377, 168, 379, 437, 438, 439, 22, 441, 442, 331, + /* 1000 */ 355, 48, 365, 50, 236, 101, 53, 364, 138, 35, + /* 1010 */ 339, 339, 369, 353, 246, 372, 0, 372, 136, 137, + /* 1020 */ 377, 408, 379, 339, 377, 412, 355, 355, 415, 416, + /* 1030 */ 417, 418, 419, 420, 233, 422, 235, 331, 331, 355, + /* 1040 */ 427, 100, 429, 372, 372, 377, 433, 434, 178, 179, + /* 1050 */ 180, 408, 99, 183, 339, 412, 372, 444, 415, 416, + /* 1060 */ 417, 418, 419, 420, 111, 422, 426, 331, 428, 22, + /* 1070 */ 355, 97, 202, 44, 426, 205, 428, 207, 208, 209, + /* 1080 */ 210, 211, 35, 377, 377, 20, 331, 372, 331, 331, + /* 1090 */ 365, 96, 345, 140, 168, 250, 143, 144, 145, 146, /* 1100 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, - /* 1110 */ 157, 158, 159, 339, 161, 162, 163, 377, 71, 249, - /* 1120 */ 339, 44, 364, 339, 185, 339, 187, 339, 401, 355, - /* 1130 */ 372, 377, 377, 44, 377, 377, 355, 379, 331, 355, - /* 1140 */ 372, 355, 248, 355, 97, 236, 372, 360, 372, 357, - /* 1150 */ 211, 383, 360, 372, 374, 246, 372, 377, 372, 383, - /* 1160 */ 372, 45, 46, 247, 248, 166, 408, 101, 0, 374, - /* 1170 */ 412, 364, 377, 415, 416, 417, 418, 419, 420, 372, - /* 1180 */ 422, 42, 44, 44, 377, 427, 379, 429, 249, 168, - /* 1190 */ 101, 433, 434, 406, 22, 374, 0, 331, 377, 42, - /* 1200 */ 44, 44, 444, 269, 168, 104, 104, 35, 107, 107, - /* 1210 */ 104, 104, 44, 107, 107, 408, 136, 137, 22, 412, - /* 1220 */ 331, 58, 415, 416, 417, 418, 419, 420, 250, 422, - /* 1230 */ 364, 353, 44, 186, 427, 188, 429, 198, 372, 200, - /* 1240 */ 433, 434, 0, 377, 35, 379, 44, 96, 1, 2, - /* 1250 */ 332, 444, 387, 364, 64, 44, 44, 101, 13, 212, - /* 1260 */ 213, 372, 469, 44, 22, 102, 377, 458, 379, 44, - /* 1270 */ 44, 44, 44, 44, 408, 44, 35, 44, 412, 44, - /* 1280 */ 35, 415, 416, 417, 418, 419, 420, 47, 422, 101, - /* 1290 */ 44, 341, 44, 427, 44, 429, 44, 408, 331, 433, - /* 1300 */ 434, 412, 44, 101, 415, 416, 417, 418, 419, 420, - /* 1310 */ 444, 422, 101, 101, 12, 13, 427, 341, 429, 168, - /* 1320 */ 101, 0, 433, 434, 22, 35, 101, 101, 101, 101, - /* 1330 */ 101, 364, 101, 444, 101, 33, 101, 35, 452, 372, - /* 1340 */ 100, 13, 387, 33, 377, 364, 379, 101, 271, 101, - /* 1350 */ 338, 101, 387, 101, 443, 376, 435, 331, 48, 101, - /* 1360 */ 58, 71, 462, 35, 54, 55, 56, 57, 58, 446, - /* 1370 */ 49, 251, 410, 71, 48, 408, 184, 409, 398, 412, - /* 1380 */ 331, 42, 415, 416, 417, 418, 419, 420, 384, 422, - /* 1390 */ 364, 20, 387, 384, 427, 165, 429, 188, 372, 97, - /* 1400 */ 433, 434, 212, 377, 382, 379, 20, 339, 339, 99, - /* 1410 */ 382, 273, 102, 364, 384, 382, 98, 95, 339, 349, - /* 1420 */ 348, 372, 347, 94, 339, 339, 377, 396, 379, 188, - /* 1430 */ 339, 20, 333, 333, 408, 20, 20, 403, 412, 345, - /* 1440 */ 379, 415, 416, 417, 418, 419, 420, 20, 422, 345, - /* 1450 */ 340, 20, 397, 427, 345, 429, 345, 408, 340, 433, - /* 1460 */ 434, 412, 331, 339, 415, 416, 417, 418, 419, 420, - /* 1470 */ 345, 422, 441, 345, 345, 52, 445, 342, 429, 169, - /* 1480 */ 170, 333, 433, 434, 174, 342, 176, 364, 186, 339, - /* 1490 */ 188, 460, 461, 333, 364, 364, 465, 466, 364, 377, - /* 1500 */ 364, 364, 192, 372, 364, 364, 364, 201, 377, 364, - /* 1510 */ 379, 331, 407, 364, 212, 213, 364, 100, 405, 343, - /* 1520 */ 403, 377, 191, 377, 343, 339, 379, 225, 226, 227, - /* 1530 */ 228, 229, 230, 231, 331, 402, 377, 258, 451, 408, - /* 1540 */ 387, 387, 257, 412, 364, 177, 415, 416, 417, 418, - /* 1550 */ 419, 420, 372, 422, 392, 377, 392, 377, 266, 379, - /* 1560 */ 429, 377, 377, 451, 433, 434, 451, 364, 454, 268, - /* 1570 */ 267, 450, 453, 470, 448, 372, 449, 252, 248, 410, - /* 1580 */ 377, 272, 379, 275, 270, 372, 20, 414, 408, 340, - /* 1590 */ 339, 20, 412, 392, 390, 415, 416, 417, 418, 419, - /* 1600 */ 420, 377, 422, 343, 343, 170, 331, 392, 343, 429, - /* 1610 */ 389, 408, 343, 433, 434, 412, 377, 100, 415, 416, - /* 1620 */ 417, 418, 419, 420, 421, 422, 423, 424, 463, 331, - /* 1630 */ 464, 377, 377, 377, 377, 372, 360, 432, 100, 364, - /* 1640 */ 368, 377, 354, 19, 339, 36, 334, 372, 333, 358, - /* 1650 */ 343, 358, 377, 358, 379, 344, 329, 33, 399, 393, - /* 1660 */ 393, 0, 364, 331, 0, 0, 404, 42, 0, 35, + /* 1110 */ 157, 158, 159, 377, 161, 162, 163, 370, 71, 249, + /* 1120 */ 339, 44, 364, 339, 339, 339, 185, 352, 187, 354, + /* 1130 */ 372, 372, 377, 387, 377, 377, 355, 379, 331, 355, + /* 1140 */ 355, 355, 383, 372, 97, 129, 130, 131, 132, 133, + /* 1150 */ 134, 135, 211, 372, 383, 360, 372, 372, 372, 357, + /* 1160 */ 44, 374, 360, 168, 377, 339, 408, 339, 0, 22, + /* 1170 */ 412, 364, 0, 415, 416, 417, 418, 419, 420, 372, + /* 1180 */ 422, 355, 35, 355, 377, 427, 379, 429, 1, 2, + /* 1190 */ 249, 433, 434, 374, 22, 44, 377, 331, 372, 44, + /* 1200 */ 372, 406, 444, 269, 42, 42, 44, 44, 104, 35, + /* 1210 */ 104, 107, 44, 107, 58, 408, 35, 101, 104, 412, + /* 1220 */ 331, 107, 415, 416, 417, 418, 419, 420, 0, 422, + /* 1230 */ 364, 166, 0, 186, 427, 188, 429, 44, 372, 104, + /* 1240 */ 433, 434, 107, 377, 198, 379, 200, 64, 332, 469, + /* 1250 */ 22, 444, 101, 364, 22, 44, 101, 458, 102, 212, + /* 1260 */ 213, 372, 44, 44, 44, 341, 377, 13, 379, 44, + /* 1270 */ 44, 44, 452, 44, 408, 44, 364, 44, 412, 44, + /* 1280 */ 341, 415, 416, 417, 418, 419, 420, 47, 422, 35, + /* 1290 */ 44, 376, 44, 427, 101, 429, 44, 408, 331, 433, + /* 1300 */ 434, 412, 273, 13, 415, 416, 417, 418, 419, 420, + /* 1310 */ 444, 422, 101, 44, 12, 13, 427, 339, 429, 101, + /* 1320 */ 101, 101, 433, 434, 22, 35, 101, 101, 101, 387, + /* 1330 */ 101, 364, 101, 444, 101, 33, 101, 35, 35, 372, + /* 1340 */ 100, 338, 387, 33, 377, 44, 379, 101, 271, 101, + /* 1350 */ 372, 443, 462, 101, 435, 446, 251, 331, 48, 410, + /* 1360 */ 58, 48, 188, 409, 54, 55, 56, 57, 58, 188, + /* 1370 */ 101, 398, 184, 71, 71, 408, 42, 384, 387, 412, + /* 1380 */ 331, 37, 415, 416, 417, 418, 419, 420, 20, 422, + /* 1390 */ 364, 384, 382, 20, 427, 212, 429, 339, 372, 97, + /* 1400 */ 433, 434, 101, 377, 165, 379, 339, 384, 349, 99, + /* 1410 */ 382, 95, 102, 364, 382, 437, 438, 439, 98, 441, + /* 1420 */ 442, 372, 348, 339, 94, 347, 377, 339, 379, 339, + /* 1430 */ 339, 20, 333, 333, 408, 20, 403, 345, 412, 20, + /* 1440 */ 379, 415, 416, 417, 418, 419, 420, 103, 422, 105, + /* 1450 */ 106, 20, 108, 427, 345, 429, 20, 408, 340, 433, + /* 1460 */ 434, 412, 331, 397, 415, 416, 417, 418, 419, 420, + /* 1470 */ 345, 422, 340, 129, 345, 345, 339, 133, 429, 169, + /* 1480 */ 170, 345, 433, 434, 174, 345, 176, 52, 186, 377, + /* 1490 */ 188, 342, 342, 333, 377, 364, 377, 364, 364, 333, + /* 1500 */ 339, 201, 192, 372, 407, 364, 364, 364, 377, 364, + /* 1510 */ 379, 331, 364, 100, 212, 213, 364, 364, 364, 343, + /* 1520 */ 405, 364, 191, 403, 402, 379, 387, 225, 226, 227, + /* 1530 */ 228, 229, 230, 231, 331, 343, 339, 258, 451, 408, + /* 1540 */ 257, 377, 377, 412, 364, 387, 415, 416, 417, 418, + /* 1550 */ 419, 420, 372, 422, 451, 377, 377, 377, 266, 379, + /* 1560 */ 429, 392, 392, 177, 433, 434, 453, 364, 275, 252, + /* 1570 */ 463, 267, 451, 268, 450, 372, 449, 454, 470, 272, + /* 1580 */ 377, 410, 379, 248, 270, 372, 448, 20, 408, 414, + /* 1590 */ 464, 340, 412, 343, 339, 415, 416, 417, 418, 419, + /* 1600 */ 420, 343, 422, 20, 390, 343, 331, 392, 392, 429, + /* 1610 */ 170, 408, 389, 433, 434, 412, 343, 372, 415, 416, + /* 1620 */ 417, 418, 419, 420, 421, 422, 423, 424, 360, 331, + /* 1630 */ 377, 100, 432, 377, 377, 100, 377, 377, 377, 364, + /* 1640 */ 368, 377, 354, 19, 36, 339, 334, 372, 333, 358, + /* 1650 */ 0, 343, 377, 404, 379, 358, 393, 33, 329, 393, + /* 1660 */ 0, 0, 364, 331, 399, 344, 0, 358, 42, 35, /* 1670 */ 372, 206, 48, 35, 35, 377, 35, 379, 54, 55, /* 1680 */ 56, 57, 58, 408, 206, 0, 35, 412, 35, 331, /* 1690 */ 415, 416, 417, 418, 419, 420, 364, 422, 206, 0, /* 1700 */ 206, 0, 35, 0, 372, 22, 408, 0, 35, 377, - /* 1710 */ 412, 379, 193, 415, 416, 417, 418, 419, 420, 188, - /* 1720 */ 422, 186, 364, 99, 0, 0, 102, 429, 0, 182, + /* 1710 */ 412, 379, 193, 415, 416, 417, 418, 419, 420, 0, + /* 1720 */ 422, 188, 364, 99, 186, 0, 102, 429, 0, 182, /* 1730 */ 372, 181, 434, 0, 0, 377, 47, 379, 0, 0, /* 1740 */ 408, 0, 467, 468, 412, 0, 42, 415, 416, 417, /* 1750 */ 418, 419, 420, 331, 422, 0, 0, 0, 134, 155, @@ -707,17 +707,17 @@ static const YYCODETYPE yy_lookahead[] = { /* 2170 */ 100, 412, 101, 35, 415, 416, 417, 418, 419, 420, /* 2180 */ 331, 422, 35, 408, 100, 35, 101, 412, 101, 100, /* 2190 */ 415, 416, 417, 418, 419, 420, 364, 422, 100, 35, - /* 2200 */ 123, 101, 100, 123, 372, 22, 112, 100, 100, 377, - /* 2210 */ 44, 379, 35, 364, 123, 123, 100, 22, 65, 64, + /* 2200 */ 122, 101, 100, 44, 372, 35, 100, 100, 122, 377, + /* 2210 */ 100, 379, 122, 364, 122, 22, 65, 64, 35, 35, /* 2220 */ 35, 372, 35, 35, 35, 35, 377, 35, 379, 331, - /* 2230 */ 71, 35, 93, 35, 35, 35, 35, 44, 35, 35, - /* 2240 */ 408, 22, 35, 35, 412, 331, 35, 415, 416, 417, - /* 2250 */ 418, 419, 420, 71, 422, 35, 35, 408, 35, 35, - /* 2260 */ 35, 412, 364, 22, 415, 416, 417, 418, 419, 420, - /* 2270 */ 372, 422, 35, 0, 35, 377, 48, 379, 364, 39, - /* 2280 */ 0, 48, 35, 39, 0, 35, 372, 39, 0, 35, - /* 2290 */ 48, 377, 39, 379, 331, 0, 35, 48, 35, 0, - /* 2300 */ 22, 22, 21, 20, 22, 471, 408, 21, 471, 471, + /* 2230 */ 35, 35, 71, 93, 35, 44, 35, 35, 22, 35, + /* 2240 */ 408, 35, 35, 71, 412, 331, 35, 415, 416, 417, + /* 2250 */ 418, 419, 420, 35, 422, 35, 35, 408, 35, 22, + /* 2260 */ 35, 412, 364, 0, 415, 416, 417, 418, 419, 420, + /* 2270 */ 372, 422, 48, 35, 39, 377, 0, 379, 364, 35, + /* 2280 */ 48, 39, 0, 35, 48, 0, 372, 35, 39, 48, + /* 2290 */ 39, 377, 0, 379, 331, 35, 35, 0, 20, 22, + /* 2300 */ 21, 471, 22, 22, 21, 471, 408, 471, 471, 471, /* 2310 */ 412, 471, 471, 415, 416, 417, 418, 419, 420, 331, /* 2320 */ 422, 471, 408, 471, 471, 471, 412, 364, 471, 415, /* 2330 */ 416, 417, 418, 419, 420, 372, 422, 471, 471, 471, @@ -755,53 +755,53 @@ static const YYCODETYPE yy_lookahead[] = { /* 2650 */ 471, 471, 471, 471, 408, 471, 471, 471, 412, 471, /* 2660 */ 471, 415, 416, 417, 418, 419, 420, 471, 422, }; -#define YY_SHIFT_COUNT (751) +#define YY_SHIFT_COUNT (749) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2299) +#define YY_SHIFT_MAX (2297) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 870, 0, 98, 0, 320, 320, 320, 320, 320, 320, /* 10 */ 320, 320, 320, 320, 320, 418, 639, 639, 737, 639, /* 20 */ 639, 639, 639, 639, 639, 639, 639, 639, 639, 639, /* 30 */ 639, 639, 639, 639, 639, 639, 639, 639, 639, 639, /* 40 */ 639, 639, 639, 639, 639, 639, 639, 639, 28, 30, - /* 50 */ 939, 17, 509, 380, 420, 380, 17, 17, 1302, 1302, - /* 60 */ 1302, 380, 1302, 1302, 190, 380, 87, 323, 186, 186, - /* 70 */ 323, 89, 89, 315, 145, 14, 14, 186, 186, 186, - /* 80 */ 186, 186, 186, 186, 221, 186, 186, 226, 87, 186, - /* 90 */ 186, 289, 186, 87, 186, 221, 186, 221, 87, 186, - /* 100 */ 186, 87, 186, 87, 87, 87, 186, 281, 953, 285, + /* 50 */ 941, 17, 599, 352, 409, 352, 17, 17, 1302, 1302, + /* 60 */ 1302, 352, 1302, 1302, 94, 352, 191, 754, 376, 376, + /* 70 */ 754, 89, 89, 213, 145, 14, 14, 376, 376, 376, + /* 80 */ 376, 376, 376, 376, 406, 376, 376, 323, 191, 376, + /* 90 */ 376, 444, 376, 191, 376, 406, 376, 406, 191, 376, + /* 100 */ 376, 191, 376, 191, 191, 191, 376, 425, 953, 285, /* 110 */ 285, 331, 150, 1047, 1047, 1047, 1047, 1047, 1047, 1047, /* 120 */ 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, 1047, - /* 130 */ 1047, 1047, 890, 594, 315, 145, 162, 171, 171, 171, - /* 140 */ 1013, 783, 783, 162, 372, 372, 372, 226, 274, 508, - /* 150 */ 87, 485, 87, 485, 485, 488, 703, 244, 244, 244, + /* 130 */ 1047, 1047, 1344, 823, 213, 145, 485, 403, 403, 403, + /* 140 */ 809, 801, 801, 485, 475, 475, 475, 323, 401, 286, + /* 150 */ 191, 516, 191, 516, 516, 504, 624, 244, 244, 244, /* 160 */ 244, 244, 244, 244, 244, 1624, 334, 473, 260, 100, - /* 170 */ 183, 25, 179, 828, 833, 326, 1021, 1116, 352, 1036, - /* 180 */ 916, 894, 934, 916, 1139, 978, 999, 1120, 1326, 1192, - /* 190 */ 1339, 1371, 1339, 1230, 1386, 1386, 1339, 1230, 1230, 1318, - /* 200 */ 1322, 1386, 1329, 1386, 1386, 1386, 1411, 1411, 1415, 226, - /* 210 */ 1416, 226, 1427, 1431, 226, 1427, 226, 226, 226, 1386, - /* 220 */ 226, 1423, 1423, 1411, 87, 87, 87, 87, 87, 87, - /* 230 */ 87, 87, 87, 87, 87, 1386, 1411, 485, 485, 485, - /* 240 */ 1306, 1417, 1415, 281, 1331, 1416, 281, 1386, 1371, 1371, - /* 250 */ 485, 1279, 1285, 485, 1279, 1285, 485, 485, 87, 1292, - /* 260 */ 1368, 1279, 1301, 1303, 1325, 1120, 1308, 1309, 1314, 1330, - /* 270 */ 372, 1566, 1386, 1427, 281, 281, 1571, 1285, 485, 485, - /* 280 */ 485, 485, 485, 1285, 485, 1435, 281, 488, 281, 372, - /* 290 */ 1517, 1538, 485, 703, 1386, 281, 1609, 1411, 2669, 2669, + /* 170 */ 183, 25, 121, 271, 558, 560, 583, 783, 547, 601, + /* 180 */ 728, 739, 934, 728, 1162, 845, 1065, 1105, 1313, 1188, + /* 190 */ 1334, 1368, 1334, 1239, 1373, 1373, 1334, 1239, 1239, 1320, + /* 200 */ 1316, 1373, 1330, 1373, 1373, 1373, 1411, 1411, 1415, 323, + /* 210 */ 1419, 323, 1431, 1436, 323, 1431, 323, 323, 323, 1373, + /* 220 */ 323, 1435, 1435, 1411, 191, 191, 191, 191, 191, 191, + /* 230 */ 191, 191, 191, 191, 191, 1373, 1411, 516, 516, 516, + /* 240 */ 1300, 1413, 1415, 425, 1331, 1419, 425, 1373, 1368, 1368, + /* 250 */ 516, 1279, 1283, 516, 1279, 1283, 516, 516, 191, 1292, + /* 260 */ 1386, 1279, 1305, 1304, 1317, 1105, 1293, 1307, 1314, 1335, + /* 270 */ 475, 1567, 1373, 1431, 425, 425, 1583, 1283, 516, 516, + /* 280 */ 516, 516, 516, 1283, 516, 1440, 425, 504, 425, 475, + /* 290 */ 1531, 1535, 516, 624, 1373, 425, 1608, 1411, 2669, 2669, /* 300 */ 2669, 2669, 2669, 2669, 2669, 2669, 2669, 170, 1310, 534, - /* 310 */ 687, 561, 580, 904, 379, 139, 374, 926, 621, 699, - /* 320 */ 699, 699, 699, 699, 699, 699, 699, 699, 173, 597, - /* 330 */ 27, 27, 439, 451, 823, 294, 419, 969, 856, 387, - /* 340 */ 791, 791, 710, 1066, 909, 710, 710, 710, 31, 1089, - /* 350 */ 1172, 1157, 588, 1168, 1101, 1102, 1106, 1107, 1073, 1196, - /* 360 */ 1242, 1039, 1156, 1188, 1163, 1202, 1211, 1212, 1080, 1077, - /* 370 */ 1138, 1151, 1219, 1225, 1226, 1227, 1228, 1229, 1247, 1231, - /* 380 */ 1209, 1241, 1190, 1233, 1240, 1235, 1246, 1248, 1250, 1252, - /* 390 */ 1258, 101, 1245, 1328, 1290, 1321, 1661, 1664, 1665, 1625, - /* 400 */ 1668, 1634, 1465, 1638, 1639, 1641, 1478, 1685, 1651, 1653, + /* 310 */ 687, 561, 580, 904, 632, 139, 788, 926, 1016, 699, + /* 320 */ 699, 699, 699, 699, 699, 699, 699, 699, 173, 559, + /* 330 */ 27, 27, 209, 230, 514, 689, 657, 974, 592, 750, + /* 340 */ 688, 688, 594, 791, 768, 594, 594, 594, 31, 1116, + /* 350 */ 1147, 1163, 130, 1168, 1104, 1106, 1114, 1135, 1172, 1228, + /* 360 */ 1232, 1046, 1151, 1155, 1156, 1193, 1211, 1218, 882, 1077, + /* 370 */ 1029, 995, 1219, 1220, 1225, 1226, 1227, 1229, 1187, 1231, + /* 380 */ 1174, 1181, 1183, 1233, 1240, 1235, 1246, 1248, 1252, 1269, + /* 390 */ 1301, 91, 1254, 1290, 1303, 898, 1650, 1660, 1661, 1626, + /* 400 */ 1666, 1634, 1465, 1638, 1639, 1641, 1478, 1685, 1651, 1653, /* 410 */ 1492, 1699, 1494, 1701, 1667, 1703, 1683, 1707, 1673, 1519, - /* 420 */ 1531, 1535, 1724, 1725, 1728, 1547, 1550, 1733, 1734, 1689, + /* 420 */ 1533, 1538, 1719, 1725, 1728, 1547, 1550, 1733, 1734, 1689, /* 430 */ 1738, 1739, 1741, 1704, 1745, 1755, 1756, 1757, 1767, 1768, /* 440 */ 1769, 1771, 1604, 1726, 1760, 1607, 1763, 1764, 1765, 1772, /* 450 */ 1779, 1781, 1782, 1783, 1784, 1787, 1788, 1790, 1791, 1792, @@ -827,14 +827,13 @@ static const unsigned short int yy_shift_ofst[] = { /* 650 */ 2047, 2049, 2050, 2055, 2057, 2103, 2059, 2060, 2109, 2061, /* 660 */ 2135, 1947, 2064, 2054, 2065, 2133, 2134, 2070, 2071, 2138, /* 670 */ 2084, 2085, 2147, 2089, 2087, 2150, 2098, 2100, 2164, 2102, - /* 680 */ 2077, 2080, 2091, 2092, 2183, 2094, 2107, 2166, 2108, 2177, - /* 690 */ 2116, 2166, 2166, 2195, 2153, 2155, 2185, 2187, 2188, 2189, - /* 700 */ 2190, 2192, 2196, 2198, 2199, 2200, 2159, 2139, 2193, 2201, - /* 710 */ 2203, 2204, 2219, 2207, 2208, 2211, 2182, 1888, 2220, 1889, - /* 720 */ 2221, 2223, 2224, 2225, 2241, 2237, 2273, 2239, 2228, 2240, - /* 730 */ 2280, 2247, 2233, 2244, 2284, 2250, 2242, 2248, 2288, 2254, - /* 740 */ 2249, 2253, 2295, 2261, 2263, 2299, 2278, 2281, 2279, 2282, - /* 750 */ 2286, 2283, + /* 680 */ 2078, 2086, 2090, 2092, 2106, 2159, 2107, 2170, 2110, 2159, + /* 690 */ 2159, 2193, 2151, 2153, 2183, 2184, 2185, 2187, 2188, 2189, + /* 700 */ 2190, 2192, 2195, 2196, 2161, 2140, 2191, 2199, 2201, 2202, + /* 710 */ 2216, 2204, 2206, 2207, 2172, 1888, 2211, 1889, 2218, 2220, + /* 720 */ 2221, 2223, 2237, 2225, 2263, 2238, 2224, 2235, 2276, 2244, + /* 730 */ 2232, 2242, 2282, 2248, 2236, 2249, 2285, 2252, 2241, 2251, + /* 740 */ 2292, 2260, 2261, 2297, 2277, 2279, 2280, 2281, 2283, 2278, }; #define YY_REDUCE_COUNT (306) #define YY_REDUCE_MIN (-436) @@ -845,110 +844,109 @@ static const short yy_reduce_ofst[] = { /* 20 */ 1358, 1422, 643, 1454, 1479, 1544, 1528, 1593, 1610, 1627, /* 30 */ 1676, 1692, 1710, 1759, 1775, 1832, 1849, 1898, 1914, 1963, /* 40 */ 1988, 2012, 2083, 2105, 2132, 2181, 2197, 2246, -316, -302, - /* 50 */ 157, 247, -256, 257, 375, 1031, 362, 391, -359, -345, - /* 60 */ -339, -436, -75, 402, -380, -178, -354, -195, 318, 332, - /* 70 */ -379, -145, 88, -282, -218, -44, 232, 116, 135, 337, - /* 80 */ 366, 388, 392, 454, -333, 542, 642, 91, -26, 645, - /* 90 */ 671, 241, 672, 236, 715, 377, 774, 427, 405, 781, - /* 100 */ 784, 535, 786, 446, 608, 572, 788, 8, -335, -19, - /* 110 */ -19, -251, -234, -163, -65, 72, 369, 461, 466, 514, - /* 120 */ 518, 547, 571, 574, 612, 706, 707, 714, 740, 754, - /* 130 */ 755, 757, 252, 23, -240, 300, 596, 23, 279, 436, - /* 140 */ 13, 182, 664, 625, 435, 768, 776, 256, 787, -417, - /* 150 */ 214, 780, -305, 795, 821, 792, 682, -364, 423, 507, - /* 160 */ 515, 533, 644, 659, 533, 727, 878, 918, 865, 793, - /* 170 */ 809, 950, 886, 981, 981, 976, 955, 1012, 979, 965, - /* 180 */ 911, 911, 900, 911, 921, 923, 981, 962, 968, 980, - /* 190 */ 1004, 1005, 1009, 1022, 1068, 1069, 1030, 1028, 1033, 1070, - /* 200 */ 1072, 1079, 1075, 1085, 1086, 1091, 1099, 1100, 1034, 1094, - /* 210 */ 1061, 1104, 1110, 1055, 1109, 1118, 1111, 1125, 1128, 1124, - /* 220 */ 1129, 1135, 1143, 1148, 1123, 1130, 1134, 1136, 1137, 1140, - /* 230 */ 1141, 1142, 1145, 1149, 1152, 1150, 1160, 1122, 1144, 1146, - /* 240 */ 1105, 1113, 1117, 1176, 1133, 1147, 1181, 1186, 1153, 1154, - /* 250 */ 1159, 1087, 1162, 1178, 1112, 1164, 1184, 1185, 981, 1114, - /* 260 */ 1119, 1115, 1121, 1127, 1126, 1169, 1103, 1166, 1165, 911, - /* 270 */ 1213, 1173, 1251, 1249, 1260, 1261, 1204, 1201, 1224, 1239, - /* 280 */ 1254, 1255, 1256, 1215, 1257, 1221, 1265, 1276, 1269, 1263, - /* 290 */ 1205, 1272, 1264, 1288, 1305, 1307, 1312, 1315, 1259, 1262, - /* 300 */ 1266, 1267, 1291, 1293, 1295, 1311, 1327, + /* 50 */ 157, 247, -256, -6, 257, 375, 556, 978, -359, -345, + /* 60 */ -339, -436, -29, 402, -380, -178, -354, -195, 354, 366, + /* 70 */ -379, -145, 1, -282, -218, 49, 171, 141, 228, 332, + /* 80 */ 460, 478, 542, 645, -333, 671, 672, 42, 110, 388, + /* 90 */ 684, 99, 715, 361, 781, -44, 784, -43, 241, 785, + /* 100 */ 786, 535, 826, 243, 608, 474, 828, 82, -335, 224, + /* 110 */ 224, -251, 22, -163, -65, 105, 342, 369, 417, 476, + /* 120 */ 545, 571, 574, 605, 612, 647, 668, 706, 707, 736, + /* 130 */ 755, 757, 205, -105, -240, 355, 380, -105, -48, 77, + /* 140 */ 118, 640, 648, 527, -276, 759, 771, 747, 795, -417, + /* 150 */ -167, 468, -305, 787, 819, 802, 775, -364, 264, 390, + /* 160 */ 445, 616, 637, 725, 616, 520, 660, 916, 746, 780, + /* 170 */ 799, 924, 820, 912, 912, 939, 942, 1003, 915, 955, + /* 180 */ 908, 908, 890, 908, 919, 909, 912, 949, 954, 973, + /* 190 */ 993, 991, 1007, 1010, 1058, 1067, 1023, 1028, 1032, 1059, + /* 200 */ 1074, 1084, 1078, 1088, 1090, 1091, 1099, 1100, 1033, 1092, + /* 210 */ 1061, 1109, 1118, 1066, 1125, 1132, 1129, 1130, 1136, 1137, + /* 220 */ 1140, 1149, 1150, 1160, 1133, 1134, 1141, 1142, 1143, 1145, + /* 230 */ 1148, 1152, 1153, 1154, 1157, 1161, 1166, 1112, 1117, 1119, + /* 240 */ 1097, 1115, 1120, 1176, 1122, 1146, 1192, 1197, 1139, 1158, + /* 250 */ 1164, 1087, 1169, 1165, 1103, 1170, 1178, 1179, 912, 1123, + /* 260 */ 1113, 1121, 1124, 1127, 1138, 1171, 1108, 1126, 1107, 908, + /* 270 */ 1213, 1175, 1255, 1251, 1250, 1258, 1214, 1215, 1253, 1256, + /* 280 */ 1257, 1259, 1260, 1216, 1261, 1223, 1262, 1268, 1273, 1245, + /* 290 */ 1200, 1272, 1264, 1288, 1306, 1308, 1312, 1315, 1265, 1249, + /* 300 */ 1263, 1266, 1291, 1297, 1309, 1321, 1329, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 10 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 20 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 30 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 40 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 50 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 60 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 70 */ 1690, 1690, 1690, 1957, 1690, 1690, 1690, 1690, 1690, 1690, - /* 80 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1769, 1690, 1690, - /* 90 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 100 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1767, 1950, 2166, - /* 110 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 120 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 130 */ 1690, 1690, 1690, 2178, 1690, 1690, 1690, 2178, 2178, 2178, - /* 140 */ 1767, 2138, 2138, 1690, 1690, 1690, 1690, 1769, 2011, 1690, - /* 150 */ 1690, 1690, 1690, 1690, 1690, 1885, 1690, 1690, 1690, 1690, - /* 160 */ 1690, 1909, 1690, 1690, 1690, 2003, 1690, 1690, 2203, 2259, - /* 170 */ 1690, 1690, 2206, 1690, 1690, 1690, 1962, 1690, 1838, 2193, - /* 180 */ 2170, 2184, 2243, 2171, 2168, 2187, 1690, 2197, 1690, 1996, - /* 190 */ 1955, 1690, 1955, 1952, 1690, 1690, 1955, 1952, 1952, 1827, - /* 200 */ 1823, 1690, 1821, 1690, 1690, 1690, 1690, 1690, 1690, 1769, - /* 210 */ 1690, 1769, 1690, 1690, 1769, 1690, 1769, 1769, 1769, 1690, - /* 220 */ 1769, 1747, 1747, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 230 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 240 */ 2023, 2009, 1690, 1767, 2005, 1690, 1767, 1690, 1690, 1690, - /* 250 */ 1690, 2214, 2212, 1690, 2214, 2212, 1690, 1690, 1690, 2228, - /* 260 */ 2224, 2214, 2232, 2230, 2199, 2197, 2262, 2249, 2245, 2184, - /* 270 */ 1690, 1690, 1690, 1690, 1767, 1767, 1690, 2212, 1690, 1690, - /* 280 */ 1690, 1690, 1690, 2212, 1690, 1690, 1767, 1690, 1767, 1690, - /* 290 */ 1690, 1854, 1690, 1690, 1690, 1767, 1722, 1690, 1998, 2014, - /* 300 */ 1980, 1980, 1888, 1888, 1888, 1770, 1695, 1690, 1690, 1690, - /* 310 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 2227, - /* 320 */ 2226, 2093, 1690, 2142, 2141, 2140, 2131, 2092, 1850, 1690, - /* 330 */ 2091, 2090, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 340 */ 1971, 1970, 2084, 1690, 1690, 2085, 2083, 2082, 1690, 1690, - /* 350 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 360 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 2246, - /* 370 */ 2250, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 2167, 1690, - /* 380 */ 1690, 1690, 1690, 1690, 2066, 1690, 1690, 1690, 1690, 1690, - /* 390 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 400 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 410 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 420 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 430 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 440 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 450 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 460 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 470 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 480 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 490 */ 1690, 1727, 2071, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 500 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 510 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 520 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 530 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 540 */ 1808, 1807, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 550 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 560 */ 1690, 1690, 1690, 1690, 2075, 1690, 1690, 1690, 1690, 1690, - /* 570 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 2242, - /* 580 */ 2200, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 590 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 2066, - /* 600 */ 1690, 2225, 1690, 1690, 2240, 1690, 2244, 1690, 1690, 1690, - /* 610 */ 1690, 1690, 1690, 1690, 2177, 2173, 1690, 1690, 2169, 1690, - /* 620 */ 1690, 1690, 1690, 1690, 1690, 1690, 2074, 1690, 1690, 1690, - /* 630 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 640 */ 2065, 1690, 2128, 1690, 1690, 1690, 2162, 1690, 1690, 2113, - /* 650 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 2075, - /* 660 */ 1690, 2078, 1690, 1690, 1690, 1690, 1690, 1882, 1690, 1690, - /* 670 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 680 */ 1867, 1865, 1864, 1863, 1690, 1860, 1690, 1895, 1690, 1690, - /* 690 */ 1690, 1891, 1890, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 700 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1788, 1690, - /* 710 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1780, 1690, 1779, - /* 720 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 730 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 740 */ 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, 1690, - /* 750 */ 1690, 1690, + /* 0 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 10 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 20 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 30 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 40 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 50 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 60 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 70 */ 1687, 1687, 1687, 1953, 1687, 1687, 1687, 1687, 1687, 1687, + /* 80 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1766, 1687, 1687, + /* 90 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 100 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1764, 1946, 2162, + /* 110 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 120 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 130 */ 1687, 1687, 1687, 2174, 1687, 1687, 1687, 2174, 2174, 2174, + /* 140 */ 1764, 2134, 2134, 1687, 1687, 1687, 1687, 1766, 2007, 1687, + /* 150 */ 1687, 1687, 1687, 1687, 1687, 1881, 1687, 1687, 1687, 1687, + /* 160 */ 1687, 1905, 1687, 1687, 1687, 1999, 1687, 1687, 2199, 2255, + /* 170 */ 1687, 1687, 2202, 1687, 1687, 1687, 1958, 1687, 1835, 2189, + /* 180 */ 2166, 2180, 2239, 2167, 2164, 2183, 1687, 2193, 1687, 1992, + /* 190 */ 1951, 1687, 1951, 1948, 1687, 1687, 1951, 1948, 1948, 1824, + /* 200 */ 1820, 1687, 1818, 1687, 1687, 1687, 1687, 1687, 1687, 1766, + /* 210 */ 1687, 1766, 1687, 1687, 1766, 1687, 1766, 1766, 1766, 1687, + /* 220 */ 1766, 1744, 1744, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 230 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 240 */ 2019, 2005, 1687, 1764, 2001, 1687, 1764, 1687, 1687, 1687, + /* 250 */ 1687, 2210, 2208, 1687, 2210, 2208, 1687, 1687, 1687, 2224, + /* 260 */ 2220, 2210, 2228, 2226, 2195, 2193, 2258, 2245, 2241, 2180, + /* 270 */ 1687, 1687, 1687, 1687, 1764, 1764, 1687, 2208, 1687, 1687, + /* 280 */ 1687, 1687, 1687, 2208, 1687, 1687, 1764, 1687, 1764, 1687, + /* 290 */ 1687, 1851, 1687, 1687, 1687, 1764, 1719, 1687, 1994, 2010, + /* 300 */ 1976, 1976, 1884, 1884, 1884, 1767, 1692, 1687, 1687, 1687, + /* 310 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 2223, + /* 320 */ 2222, 2089, 1687, 2138, 2137, 2136, 2127, 2088, 1847, 1687, + /* 330 */ 2087, 2086, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 340 */ 1967, 1966, 2080, 1687, 1687, 2081, 2079, 2078, 1687, 1687, + /* 350 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 360 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 2242, + /* 370 */ 2246, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 2163, 1687, + /* 380 */ 1687, 1687, 1687, 1687, 2062, 1687, 1687, 1687, 1687, 1687, + /* 390 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 400 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 410 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 420 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 430 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 440 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 450 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 460 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 470 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 480 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 490 */ 1687, 1724, 2067, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 500 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 510 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 520 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 530 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 540 */ 1805, 1804, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 550 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 560 */ 1687, 1687, 1687, 1687, 2071, 1687, 1687, 1687, 1687, 1687, + /* 570 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 2238, + /* 580 */ 2196, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 590 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 2062, + /* 600 */ 1687, 2221, 1687, 1687, 2236, 1687, 2240, 1687, 1687, 1687, + /* 610 */ 1687, 1687, 1687, 1687, 2173, 2169, 1687, 1687, 2165, 1687, + /* 620 */ 1687, 1687, 1687, 1687, 1687, 1687, 2070, 1687, 1687, 1687, + /* 630 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 640 */ 2061, 1687, 2124, 1687, 1687, 1687, 2158, 1687, 1687, 2109, + /* 650 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 2071, + /* 660 */ 1687, 2074, 1687, 1687, 1687, 1687, 1687, 1878, 1687, 1687, + /* 670 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 680 */ 1863, 1861, 1860, 1859, 1687, 1891, 1687, 1687, 1687, 1887, + /* 690 */ 1886, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 700 */ 1687, 1687, 1687, 1687, 1687, 1687, 1785, 1687, 1687, 1687, + /* 710 */ 1687, 1687, 1687, 1687, 1687, 1777, 1687, 1776, 1687, 1687, + /* 720 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 730 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, + /* 740 */ 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, 1687, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1080,7 +1078,6 @@ static const YYCODETYPE yyFallback[] = { 0, /* NK_EQ => nothing */ 0, /* USING => nothing */ 0, /* TAGS => nothing */ - 0, /* COMMENT => nothing */ 0, /* BOOL => nothing */ 0, /* TINYINT => nothing */ 0, /* SMALLINT => nothing */ @@ -1098,6 +1095,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* BLOB => nothing */ 0, /* VARBINARY => nothing */ 0, /* DECIMAL => nothing */ + 0, /* COMMENT => nothing */ 0, /* MAX_DELAY => nothing */ 0, /* WATERMARK => nothing */ 0, /* ROLLUP => nothing */ @@ -1495,24 +1493,24 @@ static const char *const yyTokenName[] = { /* 109 */ "NK_EQ", /* 110 */ "USING", /* 111 */ "TAGS", - /* 112 */ "COMMENT", - /* 113 */ "BOOL", - /* 114 */ "TINYINT", - /* 115 */ "SMALLINT", - /* 116 */ "INT", - /* 117 */ "INTEGER", - /* 118 */ "BIGINT", - /* 119 */ "FLOAT", - /* 120 */ "DOUBLE", - /* 121 */ "BINARY", - /* 122 */ "NCHAR", - /* 123 */ "UNSIGNED", - /* 124 */ "JSON", - /* 125 */ "VARCHAR", - /* 126 */ "MEDIUMBLOB", - /* 127 */ "BLOB", - /* 128 */ "VARBINARY", - /* 129 */ "DECIMAL", + /* 112 */ "BOOL", + /* 113 */ "TINYINT", + /* 114 */ "SMALLINT", + /* 115 */ "INT", + /* 116 */ "INTEGER", + /* 117 */ "BIGINT", + /* 118 */ "FLOAT", + /* 119 */ "DOUBLE", + /* 120 */ "BINARY", + /* 121 */ "NCHAR", + /* 122 */ "UNSIGNED", + /* 123 */ "JSON", + /* 124 */ "VARCHAR", + /* 125 */ "MEDIUMBLOB", + /* 126 */ "BLOB", + /* 127 */ "VARBINARY", + /* 128 */ "DECIMAL", + /* 129 */ "COMMENT", /* 130 */ "MAX_DELAY", /* 131 */ "WATERMARK", /* 132 */ "ROLLUP", @@ -2029,410 +2027,409 @@ static const char *const yyRuleName[] = { /* 165 */ "column_def_list ::= column_def", /* 166 */ "column_def_list ::= column_def_list NK_COMMA column_def", /* 167 */ "column_def ::= column_name type_name", - /* 168 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 169 */ "type_name ::= BOOL", - /* 170 */ "type_name ::= TINYINT", - /* 171 */ "type_name ::= SMALLINT", - /* 172 */ "type_name ::= INT", - /* 173 */ "type_name ::= INTEGER", - /* 174 */ "type_name ::= BIGINT", - /* 175 */ "type_name ::= FLOAT", - /* 176 */ "type_name ::= DOUBLE", - /* 177 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 178 */ "type_name ::= TIMESTAMP", - /* 179 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 180 */ "type_name ::= TINYINT UNSIGNED", - /* 181 */ "type_name ::= SMALLINT UNSIGNED", - /* 182 */ "type_name ::= INT UNSIGNED", - /* 183 */ "type_name ::= BIGINT UNSIGNED", - /* 184 */ "type_name ::= JSON", - /* 185 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 186 */ "type_name ::= MEDIUMBLOB", - /* 187 */ "type_name ::= BLOB", - /* 188 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 189 */ "type_name ::= DECIMAL", - /* 190 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 191 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 192 */ "tags_def_opt ::=", - /* 193 */ "tags_def_opt ::= tags_def", - /* 194 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 195 */ "table_options ::=", - /* 196 */ "table_options ::= table_options COMMENT NK_STRING", - /* 197 */ "table_options ::= table_options MAX_DELAY duration_list", - /* 198 */ "table_options ::= table_options WATERMARK duration_list", - /* 199 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", - /* 200 */ "table_options ::= table_options TTL NK_INTEGER", - /* 201 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 202 */ "table_options ::= table_options DELETE_MARK duration_list", - /* 203 */ "alter_table_options ::= alter_table_option", - /* 204 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 205 */ "alter_table_option ::= COMMENT NK_STRING", - /* 206 */ "alter_table_option ::= TTL NK_INTEGER", - /* 207 */ "duration_list ::= duration_literal", - /* 208 */ "duration_list ::= duration_list NK_COMMA duration_literal", - /* 209 */ "rollup_func_list ::= rollup_func_name", - /* 210 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", - /* 211 */ "rollup_func_name ::= function_name", - /* 212 */ "rollup_func_name ::= FIRST", - /* 213 */ "rollup_func_name ::= LAST", - /* 214 */ "col_name_list ::= col_name", - /* 215 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 216 */ "col_name ::= column_name", - /* 217 */ "cmd ::= SHOW DNODES", - /* 218 */ "cmd ::= SHOW USERS", - /* 219 */ "cmd ::= SHOW USER PRIVILEGES", - /* 220 */ "cmd ::= SHOW DATABASES", - /* 221 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 222 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 223 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 224 */ "cmd ::= SHOW MNODES", - /* 225 */ "cmd ::= SHOW QNODES", - /* 226 */ "cmd ::= SHOW FUNCTIONS", - /* 227 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 228 */ "cmd ::= SHOW STREAMS", - /* 229 */ "cmd ::= SHOW ACCOUNTS", - /* 230 */ "cmd ::= SHOW APPS", - /* 231 */ "cmd ::= SHOW CONNECTIONS", - /* 232 */ "cmd ::= SHOW LICENCES", - /* 233 */ "cmd ::= SHOW GRANTS", - /* 234 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 235 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 236 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 237 */ "cmd ::= SHOW QUERIES", - /* 238 */ "cmd ::= SHOW SCORES", - /* 239 */ "cmd ::= SHOW TOPICS", - /* 240 */ "cmd ::= SHOW VARIABLES", - /* 241 */ "cmd ::= SHOW CLUSTER VARIABLES", - /* 242 */ "cmd ::= SHOW LOCAL VARIABLES", - /* 243 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", - /* 244 */ "cmd ::= SHOW BNODES", - /* 245 */ "cmd ::= SHOW SNODES", - /* 246 */ "cmd ::= SHOW CLUSTER", - /* 247 */ "cmd ::= SHOW TRANSACTIONS", - /* 248 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", - /* 249 */ "cmd ::= SHOW CONSUMERS", - /* 250 */ "cmd ::= SHOW SUBSCRIPTIONS", - /* 251 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", - /* 252 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", - /* 253 */ "cmd ::= SHOW VNODES NK_INTEGER", - /* 254 */ "cmd ::= SHOW VNODES NK_STRING", - /* 255 */ "cmd ::= SHOW db_name_cond_opt ALIVE", - /* 256 */ "cmd ::= SHOW CLUSTER ALIVE", - /* 257 */ "db_name_cond_opt ::=", - /* 258 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 259 */ "like_pattern_opt ::=", - /* 260 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 261 */ "table_name_cond ::= table_name", - /* 262 */ "from_db_opt ::=", - /* 263 */ "from_db_opt ::= FROM db_name", - /* 264 */ "tag_list_opt ::=", - /* 265 */ "tag_list_opt ::= tag_item", - /* 266 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", - /* 267 */ "tag_item ::= TBNAME", - /* 268 */ "tag_item ::= QTAGS", - /* 269 */ "tag_item ::= column_name", - /* 270 */ "tag_item ::= column_name column_alias", - /* 271 */ "tag_item ::= column_name AS column_alias", - /* 272 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", - /* 273 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", - /* 274 */ "cmd ::= DROP INDEX exists_opt full_index_name", - /* 275 */ "full_index_name ::= index_name", - /* 276 */ "full_index_name ::= db_name NK_DOT index_name", - /* 277 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 278 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 279 */ "func_list ::= func", - /* 280 */ "func_list ::= func_list NK_COMMA func", - /* 281 */ "func ::= sma_func_name NK_LP expression_list NK_RP", - /* 282 */ "sma_func_name ::= function_name", - /* 283 */ "sma_func_name ::= COUNT", - /* 284 */ "sma_func_name ::= FIRST", - /* 285 */ "sma_func_name ::= LAST", - /* 286 */ "sma_func_name ::= LAST_ROW", - /* 287 */ "sma_stream_opt ::=", - /* 288 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", - /* 289 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", - /* 290 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", - /* 291 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", - /* 292 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 293 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 294 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 295 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", - /* 296 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 297 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 298 */ "cmd ::= DESC full_table_name", - /* 299 */ "cmd ::= DESCRIBE full_table_name", - /* 300 */ "cmd ::= RESET QUERY CACHE", - /* 301 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", - /* 302 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", - /* 303 */ "analyze_opt ::=", - /* 304 */ "analyze_opt ::= ANALYZE", - /* 305 */ "explain_options ::=", - /* 306 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 307 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 308 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", - /* 309 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 310 */ "agg_func_opt ::=", - /* 311 */ "agg_func_opt ::= AGGREGATE", - /* 312 */ "bufsize_opt ::=", - /* 313 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 314 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", - /* 315 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 316 */ "col_list_opt ::=", - /* 317 */ "col_list_opt ::= NK_LP col_name_list NK_RP", - /* 318 */ "tag_def_or_ref_opt ::=", - /* 319 */ "tag_def_or_ref_opt ::= tags_def", - /* 320 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", - /* 321 */ "stream_options ::=", - /* 322 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 323 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 324 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 325 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 326 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 327 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", - /* 328 */ "stream_options ::= stream_options DELETE_MARK duration_literal", - /* 329 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", - /* 330 */ "subtable_opt ::=", - /* 331 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", - /* 332 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 333 */ "cmd ::= KILL QUERY NK_STRING", - /* 334 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 335 */ "cmd ::= BALANCE VGROUP", - /* 336 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 337 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 338 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 339 */ "dnode_list ::= DNODE NK_INTEGER", - /* 340 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 341 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 342 */ "cmd ::= query_or_subquery", - /* 343 */ "cmd ::= insert_query", - /* 344 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", - /* 345 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", - /* 346 */ "literal ::= NK_INTEGER", - /* 347 */ "literal ::= NK_FLOAT", - /* 348 */ "literal ::= NK_STRING", - /* 349 */ "literal ::= NK_BOOL", - /* 350 */ "literal ::= TIMESTAMP NK_STRING", - /* 351 */ "literal ::= duration_literal", - /* 352 */ "literal ::= NULL", - /* 353 */ "literal ::= NK_QUESTION", - /* 354 */ "duration_literal ::= NK_VARIABLE", - /* 355 */ "signed ::= NK_INTEGER", - /* 356 */ "signed ::= NK_PLUS NK_INTEGER", - /* 357 */ "signed ::= NK_MINUS NK_INTEGER", - /* 358 */ "signed ::= NK_FLOAT", - /* 359 */ "signed ::= NK_PLUS NK_FLOAT", - /* 360 */ "signed ::= NK_MINUS NK_FLOAT", - /* 361 */ "signed_literal ::= signed", - /* 362 */ "signed_literal ::= NK_STRING", - /* 363 */ "signed_literal ::= NK_BOOL", - /* 364 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 365 */ "signed_literal ::= duration_literal", - /* 366 */ "signed_literal ::= NULL", - /* 367 */ "signed_literal ::= literal_func", - /* 368 */ "signed_literal ::= NK_QUESTION", - /* 369 */ "literal_list ::= signed_literal", - /* 370 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 371 */ "db_name ::= NK_ID", - /* 372 */ "table_name ::= NK_ID", - /* 373 */ "column_name ::= NK_ID", - /* 374 */ "function_name ::= NK_ID", - /* 375 */ "table_alias ::= NK_ID", - /* 376 */ "column_alias ::= NK_ID", - /* 377 */ "user_name ::= NK_ID", - /* 378 */ "topic_name ::= NK_ID", - /* 379 */ "stream_name ::= NK_ID", - /* 380 */ "cgroup_name ::= NK_ID", - /* 381 */ "index_name ::= NK_ID", - /* 382 */ "expr_or_subquery ::= expression", - /* 383 */ "expression ::= literal", - /* 384 */ "expression ::= pseudo_column", - /* 385 */ "expression ::= column_reference", - /* 386 */ "expression ::= function_expression", - /* 387 */ "expression ::= case_when_expression", - /* 388 */ "expression ::= NK_LP expression NK_RP", - /* 389 */ "expression ::= NK_PLUS expr_or_subquery", - /* 390 */ "expression ::= NK_MINUS expr_or_subquery", - /* 391 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", - /* 392 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", - /* 393 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", - /* 394 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", - /* 395 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", - /* 396 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 397 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", - /* 398 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", - /* 399 */ "expression_list ::= expr_or_subquery", - /* 400 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", - /* 401 */ "column_reference ::= column_name", - /* 402 */ "column_reference ::= table_name NK_DOT column_name", - /* 403 */ "pseudo_column ::= ROWTS", - /* 404 */ "pseudo_column ::= TBNAME", - /* 405 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 406 */ "pseudo_column ::= QSTART", - /* 407 */ "pseudo_column ::= QEND", - /* 408 */ "pseudo_column ::= QDURATION", - /* 409 */ "pseudo_column ::= WSTART", - /* 410 */ "pseudo_column ::= WEND", - /* 411 */ "pseudo_column ::= WDURATION", - /* 412 */ "pseudo_column ::= IROWTS", - /* 413 */ "pseudo_column ::= ISFILLED", - /* 414 */ "pseudo_column ::= QTAGS", - /* 415 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 416 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 417 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", - /* 418 */ "function_expression ::= literal_func", - /* 419 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 420 */ "literal_func ::= NOW", - /* 421 */ "noarg_func ::= NOW", - /* 422 */ "noarg_func ::= TODAY", - /* 423 */ "noarg_func ::= TIMEZONE", - /* 424 */ "noarg_func ::= DATABASE", - /* 425 */ "noarg_func ::= CLIENT_VERSION", - /* 426 */ "noarg_func ::= SERVER_VERSION", - /* 427 */ "noarg_func ::= SERVER_STATUS", - /* 428 */ "noarg_func ::= CURRENT_USER", - /* 429 */ "noarg_func ::= USER", - /* 430 */ "star_func ::= COUNT", - /* 431 */ "star_func ::= FIRST", - /* 432 */ "star_func ::= LAST", - /* 433 */ "star_func ::= LAST_ROW", - /* 434 */ "star_func_para_list ::= NK_STAR", - /* 435 */ "star_func_para_list ::= other_para_list", - /* 436 */ "other_para_list ::= star_func_para", - /* 437 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 438 */ "star_func_para ::= expr_or_subquery", - /* 439 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 440 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", - /* 441 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", - /* 442 */ "when_then_list ::= when_then_expr", - /* 443 */ "when_then_list ::= when_then_list when_then_expr", - /* 444 */ "when_then_expr ::= WHEN common_expression THEN common_expression", - /* 445 */ "case_when_else_opt ::=", - /* 446 */ "case_when_else_opt ::= ELSE common_expression", - /* 447 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", - /* 448 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", - /* 449 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", - /* 450 */ "predicate ::= expr_or_subquery IS NULL", - /* 451 */ "predicate ::= expr_or_subquery IS NOT NULL", - /* 452 */ "predicate ::= expr_or_subquery in_op in_predicate_value", - /* 453 */ "compare_op ::= NK_LT", - /* 454 */ "compare_op ::= NK_GT", - /* 455 */ "compare_op ::= NK_LE", - /* 456 */ "compare_op ::= NK_GE", - /* 457 */ "compare_op ::= NK_NE", - /* 458 */ "compare_op ::= NK_EQ", - /* 459 */ "compare_op ::= LIKE", - /* 460 */ "compare_op ::= NOT LIKE", - /* 461 */ "compare_op ::= MATCH", - /* 462 */ "compare_op ::= NMATCH", - /* 463 */ "compare_op ::= CONTAINS", - /* 464 */ "in_op ::= IN", - /* 465 */ "in_op ::= NOT IN", - /* 466 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 467 */ "boolean_value_expression ::= boolean_primary", - /* 468 */ "boolean_value_expression ::= NOT boolean_primary", - /* 469 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 470 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 471 */ "boolean_primary ::= predicate", - /* 472 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 473 */ "common_expression ::= expr_or_subquery", - /* 474 */ "common_expression ::= boolean_value_expression", - /* 475 */ "from_clause_opt ::=", - /* 476 */ "from_clause_opt ::= FROM table_reference_list", - /* 477 */ "table_reference_list ::= table_reference", - /* 478 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 479 */ "table_reference ::= table_primary", - /* 480 */ "table_reference ::= joined_table", - /* 481 */ "table_primary ::= table_name alias_opt", - /* 482 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 483 */ "table_primary ::= subquery alias_opt", - /* 484 */ "table_primary ::= parenthesized_joined_table", - /* 485 */ "alias_opt ::=", - /* 486 */ "alias_opt ::= table_alias", - /* 487 */ "alias_opt ::= AS table_alias", - /* 488 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 489 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 490 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 491 */ "join_type ::=", - /* 492 */ "join_type ::= INNER", - /* 493 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 494 */ "set_quantifier_opt ::=", - /* 495 */ "set_quantifier_opt ::= DISTINCT", - /* 496 */ "set_quantifier_opt ::= ALL", - /* 497 */ "select_list ::= select_item", - /* 498 */ "select_list ::= select_list NK_COMMA select_item", - /* 499 */ "select_item ::= NK_STAR", - /* 500 */ "select_item ::= common_expression", - /* 501 */ "select_item ::= common_expression column_alias", - /* 502 */ "select_item ::= common_expression AS column_alias", - /* 503 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 504 */ "where_clause_opt ::=", - /* 505 */ "where_clause_opt ::= WHERE search_condition", - /* 506 */ "partition_by_clause_opt ::=", - /* 507 */ "partition_by_clause_opt ::= PARTITION BY partition_list", - /* 508 */ "partition_list ::= partition_item", - /* 509 */ "partition_list ::= partition_list NK_COMMA partition_item", - /* 510 */ "partition_item ::= expr_or_subquery", - /* 511 */ "partition_item ::= expr_or_subquery column_alias", - /* 512 */ "partition_item ::= expr_or_subquery AS column_alias", - /* 513 */ "twindow_clause_opt ::=", - /* 514 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 515 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 516 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 517 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 518 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", - /* 519 */ "sliding_opt ::=", - /* 520 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 521 */ "fill_opt ::=", - /* 522 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 523 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 524 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP", - /* 525 */ "fill_mode ::= NONE", - /* 526 */ "fill_mode ::= PREV", - /* 527 */ "fill_mode ::= NULL", - /* 528 */ "fill_mode ::= NULL_F", - /* 529 */ "fill_mode ::= LINEAR", - /* 530 */ "fill_mode ::= NEXT", - /* 531 */ "group_by_clause_opt ::=", - /* 532 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 533 */ "group_by_list ::= expr_or_subquery", - /* 534 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 535 */ "having_clause_opt ::=", - /* 536 */ "having_clause_opt ::= HAVING search_condition", - /* 537 */ "range_opt ::=", - /* 538 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 539 */ "every_opt ::=", - /* 540 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 541 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 542 */ "query_simple ::= query_specification", - /* 543 */ "query_simple ::= union_query_expression", - /* 544 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 545 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 546 */ "query_simple_or_subquery ::= query_simple", - /* 547 */ "query_simple_or_subquery ::= subquery", - /* 548 */ "query_or_subquery ::= query_expression", - /* 549 */ "query_or_subquery ::= subquery", - /* 550 */ "order_by_clause_opt ::=", - /* 551 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 552 */ "slimit_clause_opt ::=", - /* 553 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 554 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 555 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 556 */ "limit_clause_opt ::=", - /* 557 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 558 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 559 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 560 */ "subquery ::= NK_LP query_expression NK_RP", - /* 561 */ "subquery ::= NK_LP subquery NK_RP", - /* 562 */ "search_condition ::= common_expression", - /* 563 */ "sort_specification_list ::= sort_specification", - /* 564 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 565 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 566 */ "ordering_specification_opt ::=", - /* 567 */ "ordering_specification_opt ::= ASC", - /* 568 */ "ordering_specification_opt ::= DESC", - /* 569 */ "null_ordering_opt ::=", - /* 570 */ "null_ordering_opt ::= NULLS FIRST", - /* 571 */ "null_ordering_opt ::= NULLS LAST", + /* 168 */ "type_name ::= BOOL", + /* 169 */ "type_name ::= TINYINT", + /* 170 */ "type_name ::= SMALLINT", + /* 171 */ "type_name ::= INT", + /* 172 */ "type_name ::= INTEGER", + /* 173 */ "type_name ::= BIGINT", + /* 174 */ "type_name ::= FLOAT", + /* 175 */ "type_name ::= DOUBLE", + /* 176 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 177 */ "type_name ::= TIMESTAMP", + /* 178 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 179 */ "type_name ::= TINYINT UNSIGNED", + /* 180 */ "type_name ::= SMALLINT UNSIGNED", + /* 181 */ "type_name ::= INT UNSIGNED", + /* 182 */ "type_name ::= BIGINT UNSIGNED", + /* 183 */ "type_name ::= JSON", + /* 184 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 185 */ "type_name ::= MEDIUMBLOB", + /* 186 */ "type_name ::= BLOB", + /* 187 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 188 */ "type_name ::= DECIMAL", + /* 189 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 190 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 191 */ "tags_def_opt ::=", + /* 192 */ "tags_def_opt ::= tags_def", + /* 193 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 194 */ "table_options ::=", + /* 195 */ "table_options ::= table_options COMMENT NK_STRING", + /* 196 */ "table_options ::= table_options MAX_DELAY duration_list", + /* 197 */ "table_options ::= table_options WATERMARK duration_list", + /* 198 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", + /* 199 */ "table_options ::= table_options TTL NK_INTEGER", + /* 200 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 201 */ "table_options ::= table_options DELETE_MARK duration_list", + /* 202 */ "alter_table_options ::= alter_table_option", + /* 203 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 204 */ "alter_table_option ::= COMMENT NK_STRING", + /* 205 */ "alter_table_option ::= TTL NK_INTEGER", + /* 206 */ "duration_list ::= duration_literal", + /* 207 */ "duration_list ::= duration_list NK_COMMA duration_literal", + /* 208 */ "rollup_func_list ::= rollup_func_name", + /* 209 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", + /* 210 */ "rollup_func_name ::= function_name", + /* 211 */ "rollup_func_name ::= FIRST", + /* 212 */ "rollup_func_name ::= LAST", + /* 213 */ "col_name_list ::= col_name", + /* 214 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 215 */ "col_name ::= column_name", + /* 216 */ "cmd ::= SHOW DNODES", + /* 217 */ "cmd ::= SHOW USERS", + /* 218 */ "cmd ::= SHOW USER PRIVILEGES", + /* 219 */ "cmd ::= SHOW DATABASES", + /* 220 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 221 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 222 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 223 */ "cmd ::= SHOW MNODES", + /* 224 */ "cmd ::= SHOW QNODES", + /* 225 */ "cmd ::= SHOW FUNCTIONS", + /* 226 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 227 */ "cmd ::= SHOW STREAMS", + /* 228 */ "cmd ::= SHOW ACCOUNTS", + /* 229 */ "cmd ::= SHOW APPS", + /* 230 */ "cmd ::= SHOW CONNECTIONS", + /* 231 */ "cmd ::= SHOW LICENCES", + /* 232 */ "cmd ::= SHOW GRANTS", + /* 233 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 234 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 235 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 236 */ "cmd ::= SHOW QUERIES", + /* 237 */ "cmd ::= SHOW SCORES", + /* 238 */ "cmd ::= SHOW TOPICS", + /* 239 */ "cmd ::= SHOW VARIABLES", + /* 240 */ "cmd ::= SHOW CLUSTER VARIABLES", + /* 241 */ "cmd ::= SHOW LOCAL VARIABLES", + /* 242 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", + /* 243 */ "cmd ::= SHOW BNODES", + /* 244 */ "cmd ::= SHOW SNODES", + /* 245 */ "cmd ::= SHOW CLUSTER", + /* 246 */ "cmd ::= SHOW TRANSACTIONS", + /* 247 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", + /* 248 */ "cmd ::= SHOW CONSUMERS", + /* 249 */ "cmd ::= SHOW SUBSCRIPTIONS", + /* 250 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", + /* 251 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", + /* 252 */ "cmd ::= SHOW VNODES NK_INTEGER", + /* 253 */ "cmd ::= SHOW VNODES NK_STRING", + /* 254 */ "cmd ::= SHOW db_name_cond_opt ALIVE", + /* 255 */ "cmd ::= SHOW CLUSTER ALIVE", + /* 256 */ "db_name_cond_opt ::=", + /* 257 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 258 */ "like_pattern_opt ::=", + /* 259 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 260 */ "table_name_cond ::= table_name", + /* 261 */ "from_db_opt ::=", + /* 262 */ "from_db_opt ::= FROM db_name", + /* 263 */ "tag_list_opt ::=", + /* 264 */ "tag_list_opt ::= tag_item", + /* 265 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", + /* 266 */ "tag_item ::= TBNAME", + /* 267 */ "tag_item ::= QTAGS", + /* 268 */ "tag_item ::= column_name", + /* 269 */ "tag_item ::= column_name column_alias", + /* 270 */ "tag_item ::= column_name AS column_alias", + /* 271 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", + /* 272 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", + /* 273 */ "cmd ::= DROP INDEX exists_opt full_index_name", + /* 274 */ "full_index_name ::= index_name", + /* 275 */ "full_index_name ::= db_name NK_DOT index_name", + /* 276 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 277 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 278 */ "func_list ::= func", + /* 279 */ "func_list ::= func_list NK_COMMA func", + /* 280 */ "func ::= sma_func_name NK_LP expression_list NK_RP", + /* 281 */ "sma_func_name ::= function_name", + /* 282 */ "sma_func_name ::= COUNT", + /* 283 */ "sma_func_name ::= FIRST", + /* 284 */ "sma_func_name ::= LAST", + /* 285 */ "sma_func_name ::= LAST_ROW", + /* 286 */ "sma_stream_opt ::=", + /* 287 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 288 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 289 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", + /* 290 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", + /* 291 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", + /* 292 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", + /* 293 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", + /* 294 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 295 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 296 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 297 */ "cmd ::= DESC full_table_name", + /* 298 */ "cmd ::= DESCRIBE full_table_name", + /* 299 */ "cmd ::= RESET QUERY CACHE", + /* 300 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", + /* 301 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", + /* 302 */ "analyze_opt ::=", + /* 303 */ "analyze_opt ::= ANALYZE", + /* 304 */ "explain_options ::=", + /* 305 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 306 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 307 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", + /* 308 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 309 */ "agg_func_opt ::=", + /* 310 */ "agg_func_opt ::= AGGREGATE", + /* 311 */ "bufsize_opt ::=", + /* 312 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 313 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", + /* 314 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 315 */ "col_list_opt ::=", + /* 316 */ "col_list_opt ::= NK_LP col_name_list NK_RP", + /* 317 */ "tag_def_or_ref_opt ::=", + /* 318 */ "tag_def_or_ref_opt ::= tags_def", + /* 319 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", + /* 320 */ "stream_options ::=", + /* 321 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 322 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 323 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 324 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 325 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 326 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", + /* 327 */ "stream_options ::= stream_options DELETE_MARK duration_literal", + /* 328 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", + /* 329 */ "subtable_opt ::=", + /* 330 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", + /* 331 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 332 */ "cmd ::= KILL QUERY NK_STRING", + /* 333 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 334 */ "cmd ::= BALANCE VGROUP", + /* 335 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 336 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 337 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 338 */ "dnode_list ::= DNODE NK_INTEGER", + /* 339 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 340 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 341 */ "cmd ::= query_or_subquery", + /* 342 */ "cmd ::= insert_query", + /* 343 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", + /* 344 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", + /* 345 */ "literal ::= NK_INTEGER", + /* 346 */ "literal ::= NK_FLOAT", + /* 347 */ "literal ::= NK_STRING", + /* 348 */ "literal ::= NK_BOOL", + /* 349 */ "literal ::= TIMESTAMP NK_STRING", + /* 350 */ "literal ::= duration_literal", + /* 351 */ "literal ::= NULL", + /* 352 */ "literal ::= NK_QUESTION", + /* 353 */ "duration_literal ::= NK_VARIABLE", + /* 354 */ "signed ::= NK_INTEGER", + /* 355 */ "signed ::= NK_PLUS NK_INTEGER", + /* 356 */ "signed ::= NK_MINUS NK_INTEGER", + /* 357 */ "signed ::= NK_FLOAT", + /* 358 */ "signed ::= NK_PLUS NK_FLOAT", + /* 359 */ "signed ::= NK_MINUS NK_FLOAT", + /* 360 */ "signed_literal ::= signed", + /* 361 */ "signed_literal ::= NK_STRING", + /* 362 */ "signed_literal ::= NK_BOOL", + /* 363 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 364 */ "signed_literal ::= duration_literal", + /* 365 */ "signed_literal ::= NULL", + /* 366 */ "signed_literal ::= literal_func", + /* 367 */ "signed_literal ::= NK_QUESTION", + /* 368 */ "literal_list ::= signed_literal", + /* 369 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 370 */ "db_name ::= NK_ID", + /* 371 */ "table_name ::= NK_ID", + /* 372 */ "column_name ::= NK_ID", + /* 373 */ "function_name ::= NK_ID", + /* 374 */ "table_alias ::= NK_ID", + /* 375 */ "column_alias ::= NK_ID", + /* 376 */ "user_name ::= NK_ID", + /* 377 */ "topic_name ::= NK_ID", + /* 378 */ "stream_name ::= NK_ID", + /* 379 */ "cgroup_name ::= NK_ID", + /* 380 */ "index_name ::= NK_ID", + /* 381 */ "expr_or_subquery ::= expression", + /* 382 */ "expression ::= literal", + /* 383 */ "expression ::= pseudo_column", + /* 384 */ "expression ::= column_reference", + /* 385 */ "expression ::= function_expression", + /* 386 */ "expression ::= case_when_expression", + /* 387 */ "expression ::= NK_LP expression NK_RP", + /* 388 */ "expression ::= NK_PLUS expr_or_subquery", + /* 389 */ "expression ::= NK_MINUS expr_or_subquery", + /* 390 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", + /* 391 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", + /* 392 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", + /* 393 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", + /* 394 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", + /* 395 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 396 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", + /* 397 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", + /* 398 */ "expression_list ::= expr_or_subquery", + /* 399 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", + /* 400 */ "column_reference ::= column_name", + /* 401 */ "column_reference ::= table_name NK_DOT column_name", + /* 402 */ "pseudo_column ::= ROWTS", + /* 403 */ "pseudo_column ::= TBNAME", + /* 404 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 405 */ "pseudo_column ::= QSTART", + /* 406 */ "pseudo_column ::= QEND", + /* 407 */ "pseudo_column ::= QDURATION", + /* 408 */ "pseudo_column ::= WSTART", + /* 409 */ "pseudo_column ::= WEND", + /* 410 */ "pseudo_column ::= WDURATION", + /* 411 */ "pseudo_column ::= IROWTS", + /* 412 */ "pseudo_column ::= ISFILLED", + /* 413 */ "pseudo_column ::= QTAGS", + /* 414 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 415 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 416 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", + /* 417 */ "function_expression ::= literal_func", + /* 418 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 419 */ "literal_func ::= NOW", + /* 420 */ "noarg_func ::= NOW", + /* 421 */ "noarg_func ::= TODAY", + /* 422 */ "noarg_func ::= TIMEZONE", + /* 423 */ "noarg_func ::= DATABASE", + /* 424 */ "noarg_func ::= CLIENT_VERSION", + /* 425 */ "noarg_func ::= SERVER_VERSION", + /* 426 */ "noarg_func ::= SERVER_STATUS", + /* 427 */ "noarg_func ::= CURRENT_USER", + /* 428 */ "noarg_func ::= USER", + /* 429 */ "star_func ::= COUNT", + /* 430 */ "star_func ::= FIRST", + /* 431 */ "star_func ::= LAST", + /* 432 */ "star_func ::= LAST_ROW", + /* 433 */ "star_func_para_list ::= NK_STAR", + /* 434 */ "star_func_para_list ::= other_para_list", + /* 435 */ "other_para_list ::= star_func_para", + /* 436 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 437 */ "star_func_para ::= expr_or_subquery", + /* 438 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 439 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", + /* 440 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", + /* 441 */ "when_then_list ::= when_then_expr", + /* 442 */ "when_then_list ::= when_then_list when_then_expr", + /* 443 */ "when_then_expr ::= WHEN common_expression THEN common_expression", + /* 444 */ "case_when_else_opt ::=", + /* 445 */ "case_when_else_opt ::= ELSE common_expression", + /* 446 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", + /* 447 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", + /* 448 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", + /* 449 */ "predicate ::= expr_or_subquery IS NULL", + /* 450 */ "predicate ::= expr_or_subquery IS NOT NULL", + /* 451 */ "predicate ::= expr_or_subquery in_op in_predicate_value", + /* 452 */ "compare_op ::= NK_LT", + /* 453 */ "compare_op ::= NK_GT", + /* 454 */ "compare_op ::= NK_LE", + /* 455 */ "compare_op ::= NK_GE", + /* 456 */ "compare_op ::= NK_NE", + /* 457 */ "compare_op ::= NK_EQ", + /* 458 */ "compare_op ::= LIKE", + /* 459 */ "compare_op ::= NOT LIKE", + /* 460 */ "compare_op ::= MATCH", + /* 461 */ "compare_op ::= NMATCH", + /* 462 */ "compare_op ::= CONTAINS", + /* 463 */ "in_op ::= IN", + /* 464 */ "in_op ::= NOT IN", + /* 465 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 466 */ "boolean_value_expression ::= boolean_primary", + /* 467 */ "boolean_value_expression ::= NOT boolean_primary", + /* 468 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 469 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 470 */ "boolean_primary ::= predicate", + /* 471 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 472 */ "common_expression ::= expr_or_subquery", + /* 473 */ "common_expression ::= boolean_value_expression", + /* 474 */ "from_clause_opt ::=", + /* 475 */ "from_clause_opt ::= FROM table_reference_list", + /* 476 */ "table_reference_list ::= table_reference", + /* 477 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 478 */ "table_reference ::= table_primary", + /* 479 */ "table_reference ::= joined_table", + /* 480 */ "table_primary ::= table_name alias_opt", + /* 481 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 482 */ "table_primary ::= subquery alias_opt", + /* 483 */ "table_primary ::= parenthesized_joined_table", + /* 484 */ "alias_opt ::=", + /* 485 */ "alias_opt ::= table_alias", + /* 486 */ "alias_opt ::= AS table_alias", + /* 487 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 488 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 489 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 490 */ "join_type ::=", + /* 491 */ "join_type ::= INNER", + /* 492 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 493 */ "set_quantifier_opt ::=", + /* 494 */ "set_quantifier_opt ::= DISTINCT", + /* 495 */ "set_quantifier_opt ::= ALL", + /* 496 */ "select_list ::= select_item", + /* 497 */ "select_list ::= select_list NK_COMMA select_item", + /* 498 */ "select_item ::= NK_STAR", + /* 499 */ "select_item ::= common_expression", + /* 500 */ "select_item ::= common_expression column_alias", + /* 501 */ "select_item ::= common_expression AS column_alias", + /* 502 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 503 */ "where_clause_opt ::=", + /* 504 */ "where_clause_opt ::= WHERE search_condition", + /* 505 */ "partition_by_clause_opt ::=", + /* 506 */ "partition_by_clause_opt ::= PARTITION BY partition_list", + /* 507 */ "partition_list ::= partition_item", + /* 508 */ "partition_list ::= partition_list NK_COMMA partition_item", + /* 509 */ "partition_item ::= expr_or_subquery", + /* 510 */ "partition_item ::= expr_or_subquery column_alias", + /* 511 */ "partition_item ::= expr_or_subquery AS column_alias", + /* 512 */ "twindow_clause_opt ::=", + /* 513 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 514 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", + /* 515 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 516 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 517 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 518 */ "sliding_opt ::=", + /* 519 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 520 */ "fill_opt ::=", + /* 521 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 522 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 523 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP", + /* 524 */ "fill_mode ::= NONE", + /* 525 */ "fill_mode ::= PREV", + /* 526 */ "fill_mode ::= NULL", + /* 527 */ "fill_mode ::= NULL_F", + /* 528 */ "fill_mode ::= LINEAR", + /* 529 */ "fill_mode ::= NEXT", + /* 530 */ "group_by_clause_opt ::=", + /* 531 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 532 */ "group_by_list ::= expr_or_subquery", + /* 533 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 534 */ "having_clause_opt ::=", + /* 535 */ "having_clause_opt ::= HAVING search_condition", + /* 536 */ "range_opt ::=", + /* 537 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 538 */ "every_opt ::=", + /* 539 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 540 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 541 */ "query_simple ::= query_specification", + /* 542 */ "query_simple ::= union_query_expression", + /* 543 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 544 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 545 */ "query_simple_or_subquery ::= query_simple", + /* 546 */ "query_simple_or_subquery ::= subquery", + /* 547 */ "query_or_subquery ::= query_expression", + /* 548 */ "query_or_subquery ::= subquery", + /* 549 */ "order_by_clause_opt ::=", + /* 550 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 551 */ "slimit_clause_opt ::=", + /* 552 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 553 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 554 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 555 */ "limit_clause_opt ::=", + /* 556 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 557 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 558 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 559 */ "subquery ::= NK_LP query_expression NK_RP", + /* 560 */ "subquery ::= NK_LP subquery NK_RP", + /* 561 */ "search_condition ::= common_expression", + /* 562 */ "sort_specification_list ::= sort_specification", + /* 563 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 564 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 565 */ "ordering_specification_opt ::=", + /* 566 */ "ordering_specification_opt ::= ASC", + /* 567 */ "ordering_specification_opt ::= DESC", + /* 568 */ "null_ordering_opt ::=", + /* 569 */ "null_ordering_opt ::= NULLS FIRST", + /* 570 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -3220,410 +3217,409 @@ static const struct { { 356, -1 }, /* (165) column_def_list ::= column_def */ { 356, -3 }, /* (166) column_def_list ::= column_def_list NK_COMMA column_def */ { 373, -2 }, /* (167) column_def ::= column_name type_name */ - { 373, -4 }, /* (168) column_def ::= column_name type_name COMMENT NK_STRING */ - { 365, -1 }, /* (169) type_name ::= BOOL */ - { 365, -1 }, /* (170) type_name ::= TINYINT */ - { 365, -1 }, /* (171) type_name ::= SMALLINT */ - { 365, -1 }, /* (172) type_name ::= INT */ - { 365, -1 }, /* (173) type_name ::= INTEGER */ - { 365, -1 }, /* (174) type_name ::= BIGINT */ - { 365, -1 }, /* (175) type_name ::= FLOAT */ - { 365, -1 }, /* (176) type_name ::= DOUBLE */ - { 365, -4 }, /* (177) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 365, -1 }, /* (178) type_name ::= TIMESTAMP */ - { 365, -4 }, /* (179) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 365, -2 }, /* (180) type_name ::= TINYINT UNSIGNED */ - { 365, -2 }, /* (181) type_name ::= SMALLINT UNSIGNED */ - { 365, -2 }, /* (182) type_name ::= INT UNSIGNED */ - { 365, -2 }, /* (183) type_name ::= BIGINT UNSIGNED */ - { 365, -1 }, /* (184) type_name ::= JSON */ - { 365, -4 }, /* (185) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 365, -1 }, /* (186) type_name ::= MEDIUMBLOB */ - { 365, -1 }, /* (187) type_name ::= BLOB */ - { 365, -4 }, /* (188) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 365, -1 }, /* (189) type_name ::= DECIMAL */ - { 365, -4 }, /* (190) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 365, -6 }, /* (191) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 357, 0 }, /* (192) tags_def_opt ::= */ - { 357, -1 }, /* (193) tags_def_opt ::= tags_def */ - { 360, -4 }, /* (194) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 358, 0 }, /* (195) table_options ::= */ - { 358, -3 }, /* (196) table_options ::= table_options COMMENT NK_STRING */ - { 358, -3 }, /* (197) table_options ::= table_options MAX_DELAY duration_list */ - { 358, -3 }, /* (198) table_options ::= table_options WATERMARK duration_list */ - { 358, -5 }, /* (199) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - { 358, -3 }, /* (200) table_options ::= table_options TTL NK_INTEGER */ - { 358, -5 }, /* (201) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 358, -3 }, /* (202) table_options ::= table_options DELETE_MARK duration_list */ - { 363, -1 }, /* (203) alter_table_options ::= alter_table_option */ - { 363, -2 }, /* (204) alter_table_options ::= alter_table_options alter_table_option */ - { 376, -2 }, /* (205) alter_table_option ::= COMMENT NK_STRING */ - { 376, -2 }, /* (206) alter_table_option ::= TTL NK_INTEGER */ - { 374, -1 }, /* (207) duration_list ::= duration_literal */ - { 374, -3 }, /* (208) duration_list ::= duration_list NK_COMMA duration_literal */ - { 375, -1 }, /* (209) rollup_func_list ::= rollup_func_name */ - { 375, -3 }, /* (210) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - { 378, -1 }, /* (211) rollup_func_name ::= function_name */ - { 378, -1 }, /* (212) rollup_func_name ::= FIRST */ - { 378, -1 }, /* (213) rollup_func_name ::= LAST */ - { 371, -1 }, /* (214) col_name_list ::= col_name */ - { 371, -3 }, /* (215) col_name_list ::= col_name_list NK_COMMA col_name */ - { 380, -1 }, /* (216) col_name ::= column_name */ - { 328, -2 }, /* (217) cmd ::= SHOW DNODES */ - { 328, -2 }, /* (218) cmd ::= SHOW USERS */ - { 328, -3 }, /* (219) cmd ::= SHOW USER PRIVILEGES */ - { 328, -2 }, /* (220) cmd ::= SHOW DATABASES */ - { 328, -4 }, /* (221) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 328, -4 }, /* (222) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 328, -3 }, /* (223) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 328, -2 }, /* (224) cmd ::= SHOW MNODES */ - { 328, -2 }, /* (225) cmd ::= SHOW QNODES */ - { 328, -2 }, /* (226) cmd ::= SHOW FUNCTIONS */ - { 328, -5 }, /* (227) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 328, -2 }, /* (228) cmd ::= SHOW STREAMS */ - { 328, -2 }, /* (229) cmd ::= SHOW ACCOUNTS */ - { 328, -2 }, /* (230) cmd ::= SHOW APPS */ - { 328, -2 }, /* (231) cmd ::= SHOW CONNECTIONS */ - { 328, -2 }, /* (232) cmd ::= SHOW LICENCES */ - { 328, -2 }, /* (233) cmd ::= SHOW GRANTS */ - { 328, -4 }, /* (234) cmd ::= SHOW CREATE DATABASE db_name */ - { 328, -4 }, /* (235) cmd ::= SHOW CREATE TABLE full_table_name */ - { 328, -4 }, /* (236) cmd ::= SHOW CREATE STABLE full_table_name */ - { 328, -2 }, /* (237) cmd ::= SHOW QUERIES */ - { 328, -2 }, /* (238) cmd ::= SHOW SCORES */ - { 328, -2 }, /* (239) cmd ::= SHOW TOPICS */ - { 328, -2 }, /* (240) cmd ::= SHOW VARIABLES */ - { 328, -3 }, /* (241) cmd ::= SHOW CLUSTER VARIABLES */ - { 328, -3 }, /* (242) cmd ::= SHOW LOCAL VARIABLES */ - { 328, -5 }, /* (243) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - { 328, -2 }, /* (244) cmd ::= SHOW BNODES */ - { 328, -2 }, /* (245) cmd ::= SHOW SNODES */ - { 328, -2 }, /* (246) cmd ::= SHOW CLUSTER */ - { 328, -2 }, /* (247) cmd ::= SHOW TRANSACTIONS */ - { 328, -4 }, /* (248) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - { 328, -2 }, /* (249) cmd ::= SHOW CONSUMERS */ - { 328, -2 }, /* (250) cmd ::= SHOW SUBSCRIPTIONS */ - { 328, -5 }, /* (251) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - { 328, -7 }, /* (252) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - { 328, -3 }, /* (253) cmd ::= SHOW VNODES NK_INTEGER */ - { 328, -3 }, /* (254) cmd ::= SHOW VNODES NK_STRING */ - { 328, -3 }, /* (255) cmd ::= SHOW db_name_cond_opt ALIVE */ - { 328, -3 }, /* (256) cmd ::= SHOW CLUSTER ALIVE */ - { 381, 0 }, /* (257) db_name_cond_opt ::= */ - { 381, -2 }, /* (258) db_name_cond_opt ::= db_name NK_DOT */ - { 382, 0 }, /* (259) like_pattern_opt ::= */ - { 382, -2 }, /* (260) like_pattern_opt ::= LIKE NK_STRING */ - { 383, -1 }, /* (261) table_name_cond ::= table_name */ - { 384, 0 }, /* (262) from_db_opt ::= */ - { 384, -2 }, /* (263) from_db_opt ::= FROM db_name */ - { 385, 0 }, /* (264) tag_list_opt ::= */ - { 385, -1 }, /* (265) tag_list_opt ::= tag_item */ - { 385, -3 }, /* (266) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - { 386, -1 }, /* (267) tag_item ::= TBNAME */ - { 386, -1 }, /* (268) tag_item ::= QTAGS */ - { 386, -1 }, /* (269) tag_item ::= column_name */ - { 386, -2 }, /* (270) tag_item ::= column_name column_alias */ - { 386, -3 }, /* (271) tag_item ::= column_name AS column_alias */ - { 328, -8 }, /* (272) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ - { 328, -9 }, /* (273) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ - { 328, -4 }, /* (274) cmd ::= DROP INDEX exists_opt full_index_name */ - { 388, -1 }, /* (275) full_index_name ::= index_name */ - { 388, -3 }, /* (276) full_index_name ::= db_name NK_DOT index_name */ - { 389, -10 }, /* (277) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - { 389, -12 }, /* (278) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - { 391, -1 }, /* (279) func_list ::= func */ - { 391, -3 }, /* (280) func_list ::= func_list NK_COMMA func */ - { 394, -4 }, /* (281) func ::= sma_func_name NK_LP expression_list NK_RP */ - { 395, -1 }, /* (282) sma_func_name ::= function_name */ - { 395, -1 }, /* (283) sma_func_name ::= COUNT */ - { 395, -1 }, /* (284) sma_func_name ::= FIRST */ - { 395, -1 }, /* (285) sma_func_name ::= LAST */ - { 395, -1 }, /* (286) sma_func_name ::= LAST_ROW */ - { 393, 0 }, /* (287) sma_stream_opt ::= */ - { 393, -3 }, /* (288) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - { 393, -3 }, /* (289) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ - { 393, -3 }, /* (290) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - { 328, -6 }, /* (291) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - { 328, -7 }, /* (292) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - { 328, -9 }, /* (293) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - { 328, -7 }, /* (294) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - { 328, -9 }, /* (295) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ - { 328, -4 }, /* (296) cmd ::= DROP TOPIC exists_opt topic_name */ - { 328, -7 }, /* (297) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - { 328, -2 }, /* (298) cmd ::= DESC full_table_name */ - { 328, -2 }, /* (299) cmd ::= DESCRIBE full_table_name */ - { 328, -3 }, /* (300) cmd ::= RESET QUERY CACHE */ - { 328, -4 }, /* (301) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - { 328, -4 }, /* (302) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ - { 398, 0 }, /* (303) analyze_opt ::= */ - { 398, -1 }, /* (304) analyze_opt ::= ANALYZE */ - { 399, 0 }, /* (305) explain_options ::= */ - { 399, -3 }, /* (306) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 399, -3 }, /* (307) explain_options ::= explain_options RATIO NK_FLOAT */ - { 328, -10 }, /* (308) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ - { 328, -4 }, /* (309) cmd ::= DROP FUNCTION exists_opt function_name */ - { 401, 0 }, /* (310) agg_func_opt ::= */ - { 401, -1 }, /* (311) agg_func_opt ::= AGGREGATE */ - { 402, 0 }, /* (312) bufsize_opt ::= */ - { 402, -2 }, /* (313) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 328, -12 }, /* (314) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ - { 328, -4 }, /* (315) cmd ::= DROP STREAM exists_opt stream_name */ - { 405, 0 }, /* (316) col_list_opt ::= */ - { 405, -3 }, /* (317) col_list_opt ::= NK_LP col_name_list NK_RP */ - { 406, 0 }, /* (318) tag_def_or_ref_opt ::= */ - { 406, -1 }, /* (319) tag_def_or_ref_opt ::= tags_def */ - { 406, -4 }, /* (320) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ - { 404, 0 }, /* (321) stream_options ::= */ - { 404, -3 }, /* (322) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 404, -3 }, /* (323) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 404, -4 }, /* (324) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 404, -3 }, /* (325) stream_options ::= stream_options WATERMARK duration_literal */ - { 404, -4 }, /* (326) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - { 404, -3 }, /* (327) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - { 404, -3 }, /* (328) stream_options ::= stream_options DELETE_MARK duration_literal */ - { 404, -4 }, /* (329) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - { 407, 0 }, /* (330) subtable_opt ::= */ - { 407, -4 }, /* (331) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - { 328, -3 }, /* (332) cmd ::= KILL CONNECTION NK_INTEGER */ - { 328, -3 }, /* (333) cmd ::= KILL QUERY NK_STRING */ - { 328, -3 }, /* (334) cmd ::= KILL TRANSACTION NK_INTEGER */ - { 328, -2 }, /* (335) cmd ::= BALANCE VGROUP */ - { 328, -4 }, /* (336) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 328, -4 }, /* (337) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 328, -3 }, /* (338) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 409, -2 }, /* (339) dnode_list ::= DNODE NK_INTEGER */ - { 409, -3 }, /* (340) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 328, -4 }, /* (341) cmd ::= DELETE FROM full_table_name where_clause_opt */ - { 328, -1 }, /* (342) cmd ::= query_or_subquery */ - { 328, -1 }, /* (343) cmd ::= insert_query */ - { 400, -7 }, /* (344) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - { 400, -4 }, /* (345) insert_query ::= INSERT INTO full_table_name query_or_subquery */ - { 331, -1 }, /* (346) literal ::= NK_INTEGER */ - { 331, -1 }, /* (347) literal ::= NK_FLOAT */ - { 331, -1 }, /* (348) literal ::= NK_STRING */ - { 331, -1 }, /* (349) literal ::= NK_BOOL */ - { 331, -2 }, /* (350) literal ::= TIMESTAMP NK_STRING */ - { 331, -1 }, /* (351) literal ::= duration_literal */ - { 331, -1 }, /* (352) literal ::= NULL */ - { 331, -1 }, /* (353) literal ::= NK_QUESTION */ - { 377, -1 }, /* (354) duration_literal ::= NK_VARIABLE */ - { 411, -1 }, /* (355) signed ::= NK_INTEGER */ - { 411, -2 }, /* (356) signed ::= NK_PLUS NK_INTEGER */ - { 411, -2 }, /* (357) signed ::= NK_MINUS NK_INTEGER */ - { 411, -1 }, /* (358) signed ::= NK_FLOAT */ - { 411, -2 }, /* (359) signed ::= NK_PLUS NK_FLOAT */ - { 411, -2 }, /* (360) signed ::= NK_MINUS NK_FLOAT */ - { 366, -1 }, /* (361) signed_literal ::= signed */ - { 366, -1 }, /* (362) signed_literal ::= NK_STRING */ - { 366, -1 }, /* (363) signed_literal ::= NK_BOOL */ - { 366, -2 }, /* (364) signed_literal ::= TIMESTAMP NK_STRING */ - { 366, -1 }, /* (365) signed_literal ::= duration_literal */ - { 366, -1 }, /* (366) signed_literal ::= NULL */ - { 366, -1 }, /* (367) signed_literal ::= literal_func */ - { 366, -1 }, /* (368) signed_literal ::= NK_QUESTION */ - { 413, -1 }, /* (369) literal_list ::= signed_literal */ - { 413, -3 }, /* (370) literal_list ::= literal_list NK_COMMA signed_literal */ - { 339, -1 }, /* (371) db_name ::= NK_ID */ - { 372, -1 }, /* (372) table_name ::= NK_ID */ - { 364, -1 }, /* (373) column_name ::= NK_ID */ - { 379, -1 }, /* (374) function_name ::= NK_ID */ - { 414, -1 }, /* (375) table_alias ::= NK_ID */ - { 387, -1 }, /* (376) column_alias ::= NK_ID */ - { 333, -1 }, /* (377) user_name ::= NK_ID */ - { 340, -1 }, /* (378) topic_name ::= NK_ID */ - { 403, -1 }, /* (379) stream_name ::= NK_ID */ - { 397, -1 }, /* (380) cgroup_name ::= NK_ID */ - { 390, -1 }, /* (381) index_name ::= NK_ID */ - { 415, -1 }, /* (382) expr_or_subquery ::= expression */ - { 408, -1 }, /* (383) expression ::= literal */ - { 408, -1 }, /* (384) expression ::= pseudo_column */ - { 408, -1 }, /* (385) expression ::= column_reference */ - { 408, -1 }, /* (386) expression ::= function_expression */ - { 408, -1 }, /* (387) expression ::= case_when_expression */ - { 408, -3 }, /* (388) expression ::= NK_LP expression NK_RP */ - { 408, -2 }, /* (389) expression ::= NK_PLUS expr_or_subquery */ - { 408, -2 }, /* (390) expression ::= NK_MINUS expr_or_subquery */ - { 408, -3 }, /* (391) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - { 408, -3 }, /* (392) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - { 408, -3 }, /* (393) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - { 408, -3 }, /* (394) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - { 408, -3 }, /* (395) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - { 408, -3 }, /* (396) expression ::= column_reference NK_ARROW NK_STRING */ - { 408, -3 }, /* (397) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - { 408, -3 }, /* (398) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - { 369, -1 }, /* (399) expression_list ::= expr_or_subquery */ - { 369, -3 }, /* (400) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - { 417, -1 }, /* (401) column_reference ::= column_name */ - { 417, -3 }, /* (402) column_reference ::= table_name NK_DOT column_name */ - { 416, -1 }, /* (403) pseudo_column ::= ROWTS */ - { 416, -1 }, /* (404) pseudo_column ::= TBNAME */ - { 416, -3 }, /* (405) pseudo_column ::= table_name NK_DOT TBNAME */ - { 416, -1 }, /* (406) pseudo_column ::= QSTART */ - { 416, -1 }, /* (407) pseudo_column ::= QEND */ - { 416, -1 }, /* (408) pseudo_column ::= QDURATION */ - { 416, -1 }, /* (409) pseudo_column ::= WSTART */ - { 416, -1 }, /* (410) pseudo_column ::= WEND */ - { 416, -1 }, /* (411) pseudo_column ::= WDURATION */ - { 416, -1 }, /* (412) pseudo_column ::= IROWTS */ - { 416, -1 }, /* (413) pseudo_column ::= ISFILLED */ - { 416, -1 }, /* (414) pseudo_column ::= QTAGS */ - { 418, -4 }, /* (415) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 418, -4 }, /* (416) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 418, -6 }, /* (417) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - { 418, -1 }, /* (418) function_expression ::= literal_func */ - { 412, -3 }, /* (419) literal_func ::= noarg_func NK_LP NK_RP */ - { 412, -1 }, /* (420) literal_func ::= NOW */ - { 422, -1 }, /* (421) noarg_func ::= NOW */ - { 422, -1 }, /* (422) noarg_func ::= TODAY */ - { 422, -1 }, /* (423) noarg_func ::= TIMEZONE */ - { 422, -1 }, /* (424) noarg_func ::= DATABASE */ - { 422, -1 }, /* (425) noarg_func ::= CLIENT_VERSION */ - { 422, -1 }, /* (426) noarg_func ::= SERVER_VERSION */ - { 422, -1 }, /* (427) noarg_func ::= SERVER_STATUS */ - { 422, -1 }, /* (428) noarg_func ::= CURRENT_USER */ - { 422, -1 }, /* (429) noarg_func ::= USER */ - { 420, -1 }, /* (430) star_func ::= COUNT */ - { 420, -1 }, /* (431) star_func ::= FIRST */ - { 420, -1 }, /* (432) star_func ::= LAST */ - { 420, -1 }, /* (433) star_func ::= LAST_ROW */ - { 421, -1 }, /* (434) star_func_para_list ::= NK_STAR */ - { 421, -1 }, /* (435) star_func_para_list ::= other_para_list */ - { 423, -1 }, /* (436) other_para_list ::= star_func_para */ - { 423, -3 }, /* (437) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 424, -1 }, /* (438) star_func_para ::= expr_or_subquery */ - { 424, -3 }, /* (439) star_func_para ::= table_name NK_DOT NK_STAR */ - { 419, -4 }, /* (440) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - { 419, -5 }, /* (441) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - { 425, -1 }, /* (442) when_then_list ::= when_then_expr */ - { 425, -2 }, /* (443) when_then_list ::= when_then_list when_then_expr */ - { 428, -4 }, /* (444) when_then_expr ::= WHEN common_expression THEN common_expression */ - { 426, 0 }, /* (445) case_when_else_opt ::= */ - { 426, -2 }, /* (446) case_when_else_opt ::= ELSE common_expression */ - { 429, -3 }, /* (447) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - { 429, -5 }, /* (448) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - { 429, -6 }, /* (449) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - { 429, -3 }, /* (450) predicate ::= expr_or_subquery IS NULL */ - { 429, -4 }, /* (451) predicate ::= expr_or_subquery IS NOT NULL */ - { 429, -3 }, /* (452) predicate ::= expr_or_subquery in_op in_predicate_value */ - { 430, -1 }, /* (453) compare_op ::= NK_LT */ - { 430, -1 }, /* (454) compare_op ::= NK_GT */ - { 430, -1 }, /* (455) compare_op ::= NK_LE */ - { 430, -1 }, /* (456) compare_op ::= NK_GE */ - { 430, -1 }, /* (457) compare_op ::= NK_NE */ - { 430, -1 }, /* (458) compare_op ::= NK_EQ */ - { 430, -1 }, /* (459) compare_op ::= LIKE */ - { 430, -2 }, /* (460) compare_op ::= NOT LIKE */ - { 430, -1 }, /* (461) compare_op ::= MATCH */ - { 430, -1 }, /* (462) compare_op ::= NMATCH */ - { 430, -1 }, /* (463) compare_op ::= CONTAINS */ - { 431, -1 }, /* (464) in_op ::= IN */ - { 431, -2 }, /* (465) in_op ::= NOT IN */ - { 432, -3 }, /* (466) in_predicate_value ::= NK_LP literal_list NK_RP */ - { 433, -1 }, /* (467) boolean_value_expression ::= boolean_primary */ - { 433, -2 }, /* (468) boolean_value_expression ::= NOT boolean_primary */ - { 433, -3 }, /* (469) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 433, -3 }, /* (470) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 434, -1 }, /* (471) boolean_primary ::= predicate */ - { 434, -3 }, /* (472) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 427, -1 }, /* (473) common_expression ::= expr_or_subquery */ - { 427, -1 }, /* (474) common_expression ::= boolean_value_expression */ - { 435, 0 }, /* (475) from_clause_opt ::= */ - { 435, -2 }, /* (476) from_clause_opt ::= FROM table_reference_list */ - { 436, -1 }, /* (477) table_reference_list ::= table_reference */ - { 436, -3 }, /* (478) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 437, -1 }, /* (479) table_reference ::= table_primary */ - { 437, -1 }, /* (480) table_reference ::= joined_table */ - { 438, -2 }, /* (481) table_primary ::= table_name alias_opt */ - { 438, -4 }, /* (482) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 438, -2 }, /* (483) table_primary ::= subquery alias_opt */ - { 438, -1 }, /* (484) table_primary ::= parenthesized_joined_table */ - { 440, 0 }, /* (485) alias_opt ::= */ - { 440, -1 }, /* (486) alias_opt ::= table_alias */ - { 440, -2 }, /* (487) alias_opt ::= AS table_alias */ - { 442, -3 }, /* (488) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 442, -3 }, /* (489) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 439, -6 }, /* (490) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 443, 0 }, /* (491) join_type ::= */ - { 443, -1 }, /* (492) join_type ::= INNER */ - { 445, -12 }, /* (493) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 446, 0 }, /* (494) set_quantifier_opt ::= */ - { 446, -1 }, /* (495) set_quantifier_opt ::= DISTINCT */ - { 446, -1 }, /* (496) set_quantifier_opt ::= ALL */ - { 447, -1 }, /* (497) select_list ::= select_item */ - { 447, -3 }, /* (498) select_list ::= select_list NK_COMMA select_item */ - { 455, -1 }, /* (499) select_item ::= NK_STAR */ - { 455, -1 }, /* (500) select_item ::= common_expression */ - { 455, -2 }, /* (501) select_item ::= common_expression column_alias */ - { 455, -3 }, /* (502) select_item ::= common_expression AS column_alias */ - { 455, -3 }, /* (503) select_item ::= table_name NK_DOT NK_STAR */ - { 410, 0 }, /* (504) where_clause_opt ::= */ - { 410, -2 }, /* (505) where_clause_opt ::= WHERE search_condition */ - { 448, 0 }, /* (506) partition_by_clause_opt ::= */ - { 448, -3 }, /* (507) partition_by_clause_opt ::= PARTITION BY partition_list */ - { 456, -1 }, /* (508) partition_list ::= partition_item */ - { 456, -3 }, /* (509) partition_list ::= partition_list NK_COMMA partition_item */ - { 457, -1 }, /* (510) partition_item ::= expr_or_subquery */ - { 457, -2 }, /* (511) partition_item ::= expr_or_subquery column_alias */ - { 457, -3 }, /* (512) partition_item ::= expr_or_subquery AS column_alias */ - { 452, 0 }, /* (513) twindow_clause_opt ::= */ - { 452, -6 }, /* (514) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 452, -4 }, /* (515) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - { 452, -6 }, /* (516) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 452, -8 }, /* (517) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 452, -7 }, /* (518) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ - { 392, 0 }, /* (519) sliding_opt ::= */ - { 392, -4 }, /* (520) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 451, 0 }, /* (521) fill_opt ::= */ - { 451, -4 }, /* (522) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 451, -6 }, /* (523) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 451, -6 }, /* (524) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ - { 458, -1 }, /* (525) fill_mode ::= NONE */ - { 458, -1 }, /* (526) fill_mode ::= PREV */ - { 458, -1 }, /* (527) fill_mode ::= NULL */ - { 458, -1 }, /* (528) fill_mode ::= NULL_F */ - { 458, -1 }, /* (529) fill_mode ::= LINEAR */ - { 458, -1 }, /* (530) fill_mode ::= NEXT */ - { 453, 0 }, /* (531) group_by_clause_opt ::= */ - { 453, -3 }, /* (532) group_by_clause_opt ::= GROUP BY group_by_list */ - { 459, -1 }, /* (533) group_by_list ::= expr_or_subquery */ - { 459, -3 }, /* (534) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - { 454, 0 }, /* (535) having_clause_opt ::= */ - { 454, -2 }, /* (536) having_clause_opt ::= HAVING search_condition */ - { 449, 0 }, /* (537) range_opt ::= */ - { 449, -6 }, /* (538) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - { 450, 0 }, /* (539) every_opt ::= */ - { 450, -4 }, /* (540) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - { 460, -4 }, /* (541) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 461, -1 }, /* (542) query_simple ::= query_specification */ - { 461, -1 }, /* (543) query_simple ::= union_query_expression */ - { 465, -4 }, /* (544) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - { 465, -3 }, /* (545) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - { 466, -1 }, /* (546) query_simple_or_subquery ::= query_simple */ - { 466, -1 }, /* (547) query_simple_or_subquery ::= subquery */ - { 396, -1 }, /* (548) query_or_subquery ::= query_expression */ - { 396, -1 }, /* (549) query_or_subquery ::= subquery */ - { 462, 0 }, /* (550) order_by_clause_opt ::= */ - { 462, -3 }, /* (551) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 463, 0 }, /* (552) slimit_clause_opt ::= */ - { 463, -2 }, /* (553) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 463, -4 }, /* (554) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 463, -4 }, /* (555) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 464, 0 }, /* (556) limit_clause_opt ::= */ - { 464, -2 }, /* (557) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 464, -4 }, /* (558) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 464, -4 }, /* (559) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 441, -3 }, /* (560) subquery ::= NK_LP query_expression NK_RP */ - { 441, -3 }, /* (561) subquery ::= NK_LP subquery NK_RP */ - { 444, -1 }, /* (562) search_condition ::= common_expression */ - { 467, -1 }, /* (563) sort_specification_list ::= sort_specification */ - { 467, -3 }, /* (564) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 468, -3 }, /* (565) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - { 469, 0 }, /* (566) ordering_specification_opt ::= */ - { 469, -1 }, /* (567) ordering_specification_opt ::= ASC */ - { 469, -1 }, /* (568) ordering_specification_opt ::= DESC */ - { 470, 0 }, /* (569) null_ordering_opt ::= */ - { 470, -2 }, /* (570) null_ordering_opt ::= NULLS FIRST */ - { 470, -2 }, /* (571) null_ordering_opt ::= NULLS LAST */ + { 365, -1 }, /* (168) type_name ::= BOOL */ + { 365, -1 }, /* (169) type_name ::= TINYINT */ + { 365, -1 }, /* (170) type_name ::= SMALLINT */ + { 365, -1 }, /* (171) type_name ::= INT */ + { 365, -1 }, /* (172) type_name ::= INTEGER */ + { 365, -1 }, /* (173) type_name ::= BIGINT */ + { 365, -1 }, /* (174) type_name ::= FLOAT */ + { 365, -1 }, /* (175) type_name ::= DOUBLE */ + { 365, -4 }, /* (176) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 365, -1 }, /* (177) type_name ::= TIMESTAMP */ + { 365, -4 }, /* (178) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 365, -2 }, /* (179) type_name ::= TINYINT UNSIGNED */ + { 365, -2 }, /* (180) type_name ::= SMALLINT UNSIGNED */ + { 365, -2 }, /* (181) type_name ::= INT UNSIGNED */ + { 365, -2 }, /* (182) type_name ::= BIGINT UNSIGNED */ + { 365, -1 }, /* (183) type_name ::= JSON */ + { 365, -4 }, /* (184) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 365, -1 }, /* (185) type_name ::= MEDIUMBLOB */ + { 365, -1 }, /* (186) type_name ::= BLOB */ + { 365, -4 }, /* (187) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 365, -1 }, /* (188) type_name ::= DECIMAL */ + { 365, -4 }, /* (189) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 365, -6 }, /* (190) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 357, 0 }, /* (191) tags_def_opt ::= */ + { 357, -1 }, /* (192) tags_def_opt ::= tags_def */ + { 360, -4 }, /* (193) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 358, 0 }, /* (194) table_options ::= */ + { 358, -3 }, /* (195) table_options ::= table_options COMMENT NK_STRING */ + { 358, -3 }, /* (196) table_options ::= table_options MAX_DELAY duration_list */ + { 358, -3 }, /* (197) table_options ::= table_options WATERMARK duration_list */ + { 358, -5 }, /* (198) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + { 358, -3 }, /* (199) table_options ::= table_options TTL NK_INTEGER */ + { 358, -5 }, /* (200) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 358, -3 }, /* (201) table_options ::= table_options DELETE_MARK duration_list */ + { 363, -1 }, /* (202) alter_table_options ::= alter_table_option */ + { 363, -2 }, /* (203) alter_table_options ::= alter_table_options alter_table_option */ + { 376, -2 }, /* (204) alter_table_option ::= COMMENT NK_STRING */ + { 376, -2 }, /* (205) alter_table_option ::= TTL NK_INTEGER */ + { 374, -1 }, /* (206) duration_list ::= duration_literal */ + { 374, -3 }, /* (207) duration_list ::= duration_list NK_COMMA duration_literal */ + { 375, -1 }, /* (208) rollup_func_list ::= rollup_func_name */ + { 375, -3 }, /* (209) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + { 378, -1 }, /* (210) rollup_func_name ::= function_name */ + { 378, -1 }, /* (211) rollup_func_name ::= FIRST */ + { 378, -1 }, /* (212) rollup_func_name ::= LAST */ + { 371, -1 }, /* (213) col_name_list ::= col_name */ + { 371, -3 }, /* (214) col_name_list ::= col_name_list NK_COMMA col_name */ + { 380, -1 }, /* (215) col_name ::= column_name */ + { 328, -2 }, /* (216) cmd ::= SHOW DNODES */ + { 328, -2 }, /* (217) cmd ::= SHOW USERS */ + { 328, -3 }, /* (218) cmd ::= SHOW USER PRIVILEGES */ + { 328, -2 }, /* (219) cmd ::= SHOW DATABASES */ + { 328, -4 }, /* (220) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 328, -4 }, /* (221) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 328, -3 }, /* (222) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 328, -2 }, /* (223) cmd ::= SHOW MNODES */ + { 328, -2 }, /* (224) cmd ::= SHOW QNODES */ + { 328, -2 }, /* (225) cmd ::= SHOW FUNCTIONS */ + { 328, -5 }, /* (226) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 328, -2 }, /* (227) cmd ::= SHOW STREAMS */ + { 328, -2 }, /* (228) cmd ::= SHOW ACCOUNTS */ + { 328, -2 }, /* (229) cmd ::= SHOW APPS */ + { 328, -2 }, /* (230) cmd ::= SHOW CONNECTIONS */ + { 328, -2 }, /* (231) cmd ::= SHOW LICENCES */ + { 328, -2 }, /* (232) cmd ::= SHOW GRANTS */ + { 328, -4 }, /* (233) cmd ::= SHOW CREATE DATABASE db_name */ + { 328, -4 }, /* (234) cmd ::= SHOW CREATE TABLE full_table_name */ + { 328, -4 }, /* (235) cmd ::= SHOW CREATE STABLE full_table_name */ + { 328, -2 }, /* (236) cmd ::= SHOW QUERIES */ + { 328, -2 }, /* (237) cmd ::= SHOW SCORES */ + { 328, -2 }, /* (238) cmd ::= SHOW TOPICS */ + { 328, -2 }, /* (239) cmd ::= SHOW VARIABLES */ + { 328, -3 }, /* (240) cmd ::= SHOW CLUSTER VARIABLES */ + { 328, -3 }, /* (241) cmd ::= SHOW LOCAL VARIABLES */ + { 328, -5 }, /* (242) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + { 328, -2 }, /* (243) cmd ::= SHOW BNODES */ + { 328, -2 }, /* (244) cmd ::= SHOW SNODES */ + { 328, -2 }, /* (245) cmd ::= SHOW CLUSTER */ + { 328, -2 }, /* (246) cmd ::= SHOW TRANSACTIONS */ + { 328, -4 }, /* (247) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + { 328, -2 }, /* (248) cmd ::= SHOW CONSUMERS */ + { 328, -2 }, /* (249) cmd ::= SHOW SUBSCRIPTIONS */ + { 328, -5 }, /* (250) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + { 328, -7 }, /* (251) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + { 328, -3 }, /* (252) cmd ::= SHOW VNODES NK_INTEGER */ + { 328, -3 }, /* (253) cmd ::= SHOW VNODES NK_STRING */ + { 328, -3 }, /* (254) cmd ::= SHOW db_name_cond_opt ALIVE */ + { 328, -3 }, /* (255) cmd ::= SHOW CLUSTER ALIVE */ + { 381, 0 }, /* (256) db_name_cond_opt ::= */ + { 381, -2 }, /* (257) db_name_cond_opt ::= db_name NK_DOT */ + { 382, 0 }, /* (258) like_pattern_opt ::= */ + { 382, -2 }, /* (259) like_pattern_opt ::= LIKE NK_STRING */ + { 383, -1 }, /* (260) table_name_cond ::= table_name */ + { 384, 0 }, /* (261) from_db_opt ::= */ + { 384, -2 }, /* (262) from_db_opt ::= FROM db_name */ + { 385, 0 }, /* (263) tag_list_opt ::= */ + { 385, -1 }, /* (264) tag_list_opt ::= tag_item */ + { 385, -3 }, /* (265) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + { 386, -1 }, /* (266) tag_item ::= TBNAME */ + { 386, -1 }, /* (267) tag_item ::= QTAGS */ + { 386, -1 }, /* (268) tag_item ::= column_name */ + { 386, -2 }, /* (269) tag_item ::= column_name column_alias */ + { 386, -3 }, /* (270) tag_item ::= column_name AS column_alias */ + { 328, -8 }, /* (271) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + { 328, -9 }, /* (272) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + { 328, -4 }, /* (273) cmd ::= DROP INDEX exists_opt full_index_name */ + { 388, -1 }, /* (274) full_index_name ::= index_name */ + { 388, -3 }, /* (275) full_index_name ::= db_name NK_DOT index_name */ + { 389, -10 }, /* (276) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + { 389, -12 }, /* (277) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + { 391, -1 }, /* (278) func_list ::= func */ + { 391, -3 }, /* (279) func_list ::= func_list NK_COMMA func */ + { 394, -4 }, /* (280) func ::= sma_func_name NK_LP expression_list NK_RP */ + { 395, -1 }, /* (281) sma_func_name ::= function_name */ + { 395, -1 }, /* (282) sma_func_name ::= COUNT */ + { 395, -1 }, /* (283) sma_func_name ::= FIRST */ + { 395, -1 }, /* (284) sma_func_name ::= LAST */ + { 395, -1 }, /* (285) sma_func_name ::= LAST_ROW */ + { 393, 0 }, /* (286) sma_stream_opt ::= */ + { 393, -3 }, /* (287) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + { 393, -3 }, /* (288) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + { 393, -3 }, /* (289) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + { 328, -6 }, /* (290) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + { 328, -7 }, /* (291) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ + { 328, -9 }, /* (292) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ + { 328, -7 }, /* (293) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ + { 328, -9 }, /* (294) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + { 328, -4 }, /* (295) cmd ::= DROP TOPIC exists_opt topic_name */ + { 328, -7 }, /* (296) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + { 328, -2 }, /* (297) cmd ::= DESC full_table_name */ + { 328, -2 }, /* (298) cmd ::= DESCRIBE full_table_name */ + { 328, -3 }, /* (299) cmd ::= RESET QUERY CACHE */ + { 328, -4 }, /* (300) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + { 328, -4 }, /* (301) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ + { 398, 0 }, /* (302) analyze_opt ::= */ + { 398, -1 }, /* (303) analyze_opt ::= ANALYZE */ + { 399, 0 }, /* (304) explain_options ::= */ + { 399, -3 }, /* (305) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 399, -3 }, /* (306) explain_options ::= explain_options RATIO NK_FLOAT */ + { 328, -10 }, /* (307) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ + { 328, -4 }, /* (308) cmd ::= DROP FUNCTION exists_opt function_name */ + { 401, 0 }, /* (309) agg_func_opt ::= */ + { 401, -1 }, /* (310) agg_func_opt ::= AGGREGATE */ + { 402, 0 }, /* (311) bufsize_opt ::= */ + { 402, -2 }, /* (312) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 328, -12 }, /* (313) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ + { 328, -4 }, /* (314) cmd ::= DROP STREAM exists_opt stream_name */ + { 405, 0 }, /* (315) col_list_opt ::= */ + { 405, -3 }, /* (316) col_list_opt ::= NK_LP col_name_list NK_RP */ + { 406, 0 }, /* (317) tag_def_or_ref_opt ::= */ + { 406, -1 }, /* (318) tag_def_or_ref_opt ::= tags_def */ + { 406, -4 }, /* (319) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + { 404, 0 }, /* (320) stream_options ::= */ + { 404, -3 }, /* (321) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 404, -3 }, /* (322) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 404, -4 }, /* (323) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 404, -3 }, /* (324) stream_options ::= stream_options WATERMARK duration_literal */ + { 404, -4 }, /* (325) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + { 404, -3 }, /* (326) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + { 404, -3 }, /* (327) stream_options ::= stream_options DELETE_MARK duration_literal */ + { 404, -4 }, /* (328) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + { 407, 0 }, /* (329) subtable_opt ::= */ + { 407, -4 }, /* (330) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + { 328, -3 }, /* (331) cmd ::= KILL CONNECTION NK_INTEGER */ + { 328, -3 }, /* (332) cmd ::= KILL QUERY NK_STRING */ + { 328, -3 }, /* (333) cmd ::= KILL TRANSACTION NK_INTEGER */ + { 328, -2 }, /* (334) cmd ::= BALANCE VGROUP */ + { 328, -4 }, /* (335) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 328, -4 }, /* (336) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 328, -3 }, /* (337) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 409, -2 }, /* (338) dnode_list ::= DNODE NK_INTEGER */ + { 409, -3 }, /* (339) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 328, -4 }, /* (340) cmd ::= DELETE FROM full_table_name where_clause_opt */ + { 328, -1 }, /* (341) cmd ::= query_or_subquery */ + { 328, -1 }, /* (342) cmd ::= insert_query */ + { 400, -7 }, /* (343) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + { 400, -4 }, /* (344) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + { 331, -1 }, /* (345) literal ::= NK_INTEGER */ + { 331, -1 }, /* (346) literal ::= NK_FLOAT */ + { 331, -1 }, /* (347) literal ::= NK_STRING */ + { 331, -1 }, /* (348) literal ::= NK_BOOL */ + { 331, -2 }, /* (349) literal ::= TIMESTAMP NK_STRING */ + { 331, -1 }, /* (350) literal ::= duration_literal */ + { 331, -1 }, /* (351) literal ::= NULL */ + { 331, -1 }, /* (352) literal ::= NK_QUESTION */ + { 377, -1 }, /* (353) duration_literal ::= NK_VARIABLE */ + { 411, -1 }, /* (354) signed ::= NK_INTEGER */ + { 411, -2 }, /* (355) signed ::= NK_PLUS NK_INTEGER */ + { 411, -2 }, /* (356) signed ::= NK_MINUS NK_INTEGER */ + { 411, -1 }, /* (357) signed ::= NK_FLOAT */ + { 411, -2 }, /* (358) signed ::= NK_PLUS NK_FLOAT */ + { 411, -2 }, /* (359) signed ::= NK_MINUS NK_FLOAT */ + { 366, -1 }, /* (360) signed_literal ::= signed */ + { 366, -1 }, /* (361) signed_literal ::= NK_STRING */ + { 366, -1 }, /* (362) signed_literal ::= NK_BOOL */ + { 366, -2 }, /* (363) signed_literal ::= TIMESTAMP NK_STRING */ + { 366, -1 }, /* (364) signed_literal ::= duration_literal */ + { 366, -1 }, /* (365) signed_literal ::= NULL */ + { 366, -1 }, /* (366) signed_literal ::= literal_func */ + { 366, -1 }, /* (367) signed_literal ::= NK_QUESTION */ + { 413, -1 }, /* (368) literal_list ::= signed_literal */ + { 413, -3 }, /* (369) literal_list ::= literal_list NK_COMMA signed_literal */ + { 339, -1 }, /* (370) db_name ::= NK_ID */ + { 372, -1 }, /* (371) table_name ::= NK_ID */ + { 364, -1 }, /* (372) column_name ::= NK_ID */ + { 379, -1 }, /* (373) function_name ::= NK_ID */ + { 414, -1 }, /* (374) table_alias ::= NK_ID */ + { 387, -1 }, /* (375) column_alias ::= NK_ID */ + { 333, -1 }, /* (376) user_name ::= NK_ID */ + { 340, -1 }, /* (377) topic_name ::= NK_ID */ + { 403, -1 }, /* (378) stream_name ::= NK_ID */ + { 397, -1 }, /* (379) cgroup_name ::= NK_ID */ + { 390, -1 }, /* (380) index_name ::= NK_ID */ + { 415, -1 }, /* (381) expr_or_subquery ::= expression */ + { 408, -1 }, /* (382) expression ::= literal */ + { 408, -1 }, /* (383) expression ::= pseudo_column */ + { 408, -1 }, /* (384) expression ::= column_reference */ + { 408, -1 }, /* (385) expression ::= function_expression */ + { 408, -1 }, /* (386) expression ::= case_when_expression */ + { 408, -3 }, /* (387) expression ::= NK_LP expression NK_RP */ + { 408, -2 }, /* (388) expression ::= NK_PLUS expr_or_subquery */ + { 408, -2 }, /* (389) expression ::= NK_MINUS expr_or_subquery */ + { 408, -3 }, /* (390) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + { 408, -3 }, /* (391) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + { 408, -3 }, /* (392) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + { 408, -3 }, /* (393) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + { 408, -3 }, /* (394) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + { 408, -3 }, /* (395) expression ::= column_reference NK_ARROW NK_STRING */ + { 408, -3 }, /* (396) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + { 408, -3 }, /* (397) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + { 369, -1 }, /* (398) expression_list ::= expr_or_subquery */ + { 369, -3 }, /* (399) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + { 417, -1 }, /* (400) column_reference ::= column_name */ + { 417, -3 }, /* (401) column_reference ::= table_name NK_DOT column_name */ + { 416, -1 }, /* (402) pseudo_column ::= ROWTS */ + { 416, -1 }, /* (403) pseudo_column ::= TBNAME */ + { 416, -3 }, /* (404) pseudo_column ::= table_name NK_DOT TBNAME */ + { 416, -1 }, /* (405) pseudo_column ::= QSTART */ + { 416, -1 }, /* (406) pseudo_column ::= QEND */ + { 416, -1 }, /* (407) pseudo_column ::= QDURATION */ + { 416, -1 }, /* (408) pseudo_column ::= WSTART */ + { 416, -1 }, /* (409) pseudo_column ::= WEND */ + { 416, -1 }, /* (410) pseudo_column ::= WDURATION */ + { 416, -1 }, /* (411) pseudo_column ::= IROWTS */ + { 416, -1 }, /* (412) pseudo_column ::= ISFILLED */ + { 416, -1 }, /* (413) pseudo_column ::= QTAGS */ + { 418, -4 }, /* (414) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 418, -4 }, /* (415) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 418, -6 }, /* (416) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + { 418, -1 }, /* (417) function_expression ::= literal_func */ + { 412, -3 }, /* (418) literal_func ::= noarg_func NK_LP NK_RP */ + { 412, -1 }, /* (419) literal_func ::= NOW */ + { 422, -1 }, /* (420) noarg_func ::= NOW */ + { 422, -1 }, /* (421) noarg_func ::= TODAY */ + { 422, -1 }, /* (422) noarg_func ::= TIMEZONE */ + { 422, -1 }, /* (423) noarg_func ::= DATABASE */ + { 422, -1 }, /* (424) noarg_func ::= CLIENT_VERSION */ + { 422, -1 }, /* (425) noarg_func ::= SERVER_VERSION */ + { 422, -1 }, /* (426) noarg_func ::= SERVER_STATUS */ + { 422, -1 }, /* (427) noarg_func ::= CURRENT_USER */ + { 422, -1 }, /* (428) noarg_func ::= USER */ + { 420, -1 }, /* (429) star_func ::= COUNT */ + { 420, -1 }, /* (430) star_func ::= FIRST */ + { 420, -1 }, /* (431) star_func ::= LAST */ + { 420, -1 }, /* (432) star_func ::= LAST_ROW */ + { 421, -1 }, /* (433) star_func_para_list ::= NK_STAR */ + { 421, -1 }, /* (434) star_func_para_list ::= other_para_list */ + { 423, -1 }, /* (435) other_para_list ::= star_func_para */ + { 423, -3 }, /* (436) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 424, -1 }, /* (437) star_func_para ::= expr_or_subquery */ + { 424, -3 }, /* (438) star_func_para ::= table_name NK_DOT NK_STAR */ + { 419, -4 }, /* (439) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + { 419, -5 }, /* (440) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + { 425, -1 }, /* (441) when_then_list ::= when_then_expr */ + { 425, -2 }, /* (442) when_then_list ::= when_then_list when_then_expr */ + { 428, -4 }, /* (443) when_then_expr ::= WHEN common_expression THEN common_expression */ + { 426, 0 }, /* (444) case_when_else_opt ::= */ + { 426, -2 }, /* (445) case_when_else_opt ::= ELSE common_expression */ + { 429, -3 }, /* (446) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + { 429, -5 }, /* (447) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + { 429, -6 }, /* (448) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + { 429, -3 }, /* (449) predicate ::= expr_or_subquery IS NULL */ + { 429, -4 }, /* (450) predicate ::= expr_or_subquery IS NOT NULL */ + { 429, -3 }, /* (451) predicate ::= expr_or_subquery in_op in_predicate_value */ + { 430, -1 }, /* (452) compare_op ::= NK_LT */ + { 430, -1 }, /* (453) compare_op ::= NK_GT */ + { 430, -1 }, /* (454) compare_op ::= NK_LE */ + { 430, -1 }, /* (455) compare_op ::= NK_GE */ + { 430, -1 }, /* (456) compare_op ::= NK_NE */ + { 430, -1 }, /* (457) compare_op ::= NK_EQ */ + { 430, -1 }, /* (458) compare_op ::= LIKE */ + { 430, -2 }, /* (459) compare_op ::= NOT LIKE */ + { 430, -1 }, /* (460) compare_op ::= MATCH */ + { 430, -1 }, /* (461) compare_op ::= NMATCH */ + { 430, -1 }, /* (462) compare_op ::= CONTAINS */ + { 431, -1 }, /* (463) in_op ::= IN */ + { 431, -2 }, /* (464) in_op ::= NOT IN */ + { 432, -3 }, /* (465) in_predicate_value ::= NK_LP literal_list NK_RP */ + { 433, -1 }, /* (466) boolean_value_expression ::= boolean_primary */ + { 433, -2 }, /* (467) boolean_value_expression ::= NOT boolean_primary */ + { 433, -3 }, /* (468) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 433, -3 }, /* (469) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 434, -1 }, /* (470) boolean_primary ::= predicate */ + { 434, -3 }, /* (471) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 427, -1 }, /* (472) common_expression ::= expr_or_subquery */ + { 427, -1 }, /* (473) common_expression ::= boolean_value_expression */ + { 435, 0 }, /* (474) from_clause_opt ::= */ + { 435, -2 }, /* (475) from_clause_opt ::= FROM table_reference_list */ + { 436, -1 }, /* (476) table_reference_list ::= table_reference */ + { 436, -3 }, /* (477) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 437, -1 }, /* (478) table_reference ::= table_primary */ + { 437, -1 }, /* (479) table_reference ::= joined_table */ + { 438, -2 }, /* (480) table_primary ::= table_name alias_opt */ + { 438, -4 }, /* (481) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 438, -2 }, /* (482) table_primary ::= subquery alias_opt */ + { 438, -1 }, /* (483) table_primary ::= parenthesized_joined_table */ + { 440, 0 }, /* (484) alias_opt ::= */ + { 440, -1 }, /* (485) alias_opt ::= table_alias */ + { 440, -2 }, /* (486) alias_opt ::= AS table_alias */ + { 442, -3 }, /* (487) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 442, -3 }, /* (488) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 439, -6 }, /* (489) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 443, 0 }, /* (490) join_type ::= */ + { 443, -1 }, /* (491) join_type ::= INNER */ + { 445, -12 }, /* (492) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 446, 0 }, /* (493) set_quantifier_opt ::= */ + { 446, -1 }, /* (494) set_quantifier_opt ::= DISTINCT */ + { 446, -1 }, /* (495) set_quantifier_opt ::= ALL */ + { 447, -1 }, /* (496) select_list ::= select_item */ + { 447, -3 }, /* (497) select_list ::= select_list NK_COMMA select_item */ + { 455, -1 }, /* (498) select_item ::= NK_STAR */ + { 455, -1 }, /* (499) select_item ::= common_expression */ + { 455, -2 }, /* (500) select_item ::= common_expression column_alias */ + { 455, -3 }, /* (501) select_item ::= common_expression AS column_alias */ + { 455, -3 }, /* (502) select_item ::= table_name NK_DOT NK_STAR */ + { 410, 0 }, /* (503) where_clause_opt ::= */ + { 410, -2 }, /* (504) where_clause_opt ::= WHERE search_condition */ + { 448, 0 }, /* (505) partition_by_clause_opt ::= */ + { 448, -3 }, /* (506) partition_by_clause_opt ::= PARTITION BY partition_list */ + { 456, -1 }, /* (507) partition_list ::= partition_item */ + { 456, -3 }, /* (508) partition_list ::= partition_list NK_COMMA partition_item */ + { 457, -1 }, /* (509) partition_item ::= expr_or_subquery */ + { 457, -2 }, /* (510) partition_item ::= expr_or_subquery column_alias */ + { 457, -3 }, /* (511) partition_item ::= expr_or_subquery AS column_alias */ + { 452, 0 }, /* (512) twindow_clause_opt ::= */ + { 452, -6 }, /* (513) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 452, -4 }, /* (514) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + { 452, -6 }, /* (515) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 452, -8 }, /* (516) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 452, -7 }, /* (517) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + { 392, 0 }, /* (518) sliding_opt ::= */ + { 392, -4 }, /* (519) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 451, 0 }, /* (520) fill_opt ::= */ + { 451, -4 }, /* (521) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 451, -6 }, /* (522) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 451, -6 }, /* (523) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ + { 458, -1 }, /* (524) fill_mode ::= NONE */ + { 458, -1 }, /* (525) fill_mode ::= PREV */ + { 458, -1 }, /* (526) fill_mode ::= NULL */ + { 458, -1 }, /* (527) fill_mode ::= NULL_F */ + { 458, -1 }, /* (528) fill_mode ::= LINEAR */ + { 458, -1 }, /* (529) fill_mode ::= NEXT */ + { 453, 0 }, /* (530) group_by_clause_opt ::= */ + { 453, -3 }, /* (531) group_by_clause_opt ::= GROUP BY group_by_list */ + { 459, -1 }, /* (532) group_by_list ::= expr_or_subquery */ + { 459, -3 }, /* (533) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + { 454, 0 }, /* (534) having_clause_opt ::= */ + { 454, -2 }, /* (535) having_clause_opt ::= HAVING search_condition */ + { 449, 0 }, /* (536) range_opt ::= */ + { 449, -6 }, /* (537) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + { 450, 0 }, /* (538) every_opt ::= */ + { 450, -4 }, /* (539) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + { 460, -4 }, /* (540) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 461, -1 }, /* (541) query_simple ::= query_specification */ + { 461, -1 }, /* (542) query_simple ::= union_query_expression */ + { 465, -4 }, /* (543) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + { 465, -3 }, /* (544) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + { 466, -1 }, /* (545) query_simple_or_subquery ::= query_simple */ + { 466, -1 }, /* (546) query_simple_or_subquery ::= subquery */ + { 396, -1 }, /* (547) query_or_subquery ::= query_expression */ + { 396, -1 }, /* (548) query_or_subquery ::= subquery */ + { 462, 0 }, /* (549) order_by_clause_opt ::= */ + { 462, -3 }, /* (550) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 463, 0 }, /* (551) slimit_clause_opt ::= */ + { 463, -2 }, /* (552) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 463, -4 }, /* (553) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 463, -4 }, /* (554) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 464, 0 }, /* (555) limit_clause_opt ::= */ + { 464, -2 }, /* (556) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 464, -4 }, /* (557) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 464, -4 }, /* (558) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 441, -3 }, /* (559) subquery ::= NK_LP query_expression NK_RP */ + { 441, -3 }, /* (560) subquery ::= NK_LP subquery NK_RP */ + { 444, -1 }, /* (561) search_condition ::= common_expression */ + { 467, -1 }, /* (562) sort_specification_list ::= sort_specification */ + { 467, -3 }, /* (563) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 468, -3 }, /* (564) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + { 469, 0 }, /* (565) ordering_specification_opt ::= */ + { 469, -1 }, /* (566) ordering_specification_opt ::= ASC */ + { 469, -1 }, /* (567) ordering_specification_opt ::= DESC */ + { 470, 0 }, /* (568) null_ordering_opt ::= */ + { 470, -2 }, /* (569) null_ordering_opt ::= NULLS FIRST */ + { 470, -2 }, /* (570) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3816,8 +3812,8 @@ static YYACTIONTYPE yy_reduce( yymsp[-2].minor.yy881 = yylhsminor.yy881; break; case 42: /* priv_level ::= topic_name */ - case 282: /* sma_func_name ::= function_name */ yytestcase(yyruleno==282); - case 486: /* alias_opt ::= table_alias */ yytestcase(yyruleno==486); + case 281: /* sma_func_name ::= function_name */ yytestcase(yyruleno==281); + case 485: /* alias_opt ::= table_alias */ yytestcase(yyruleno==485); { yylhsminor.yy881 = yymsp[0].minor.yy881; } yymsp[0].minor.yy881 = yylhsminor.yy881; break; @@ -3848,49 +3844,49 @@ static YYACTIONTYPE yy_reduce( case 51: /* dnode_endpoint ::= NK_STRING */ case 52: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==52); case 53: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==53); - case 283: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==283); - case 284: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==284); - case 285: /* sma_func_name ::= LAST */ yytestcase(yyruleno==285); - case 286: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==286); - case 371: /* db_name ::= NK_ID */ yytestcase(yyruleno==371); - case 372: /* table_name ::= NK_ID */ yytestcase(yyruleno==372); - case 373: /* column_name ::= NK_ID */ yytestcase(yyruleno==373); - case 374: /* function_name ::= NK_ID */ yytestcase(yyruleno==374); - case 375: /* table_alias ::= NK_ID */ yytestcase(yyruleno==375); - case 376: /* column_alias ::= NK_ID */ yytestcase(yyruleno==376); - case 377: /* user_name ::= NK_ID */ yytestcase(yyruleno==377); - case 378: /* topic_name ::= NK_ID */ yytestcase(yyruleno==378); - case 379: /* stream_name ::= NK_ID */ yytestcase(yyruleno==379); - case 380: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==380); - case 381: /* index_name ::= NK_ID */ yytestcase(yyruleno==381); - case 421: /* noarg_func ::= NOW */ yytestcase(yyruleno==421); - case 422: /* noarg_func ::= TODAY */ yytestcase(yyruleno==422); - case 423: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==423); - case 424: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==424); - case 425: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==425); - case 426: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==426); - case 427: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==427); - case 428: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==428); - case 429: /* noarg_func ::= USER */ yytestcase(yyruleno==429); - case 430: /* star_func ::= COUNT */ yytestcase(yyruleno==430); - case 431: /* star_func ::= FIRST */ yytestcase(yyruleno==431); - case 432: /* star_func ::= LAST */ yytestcase(yyruleno==432); - case 433: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==433); + case 282: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==282); + case 283: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==283); + case 284: /* sma_func_name ::= LAST */ yytestcase(yyruleno==284); + case 285: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==285); + case 370: /* db_name ::= NK_ID */ yytestcase(yyruleno==370); + case 371: /* table_name ::= NK_ID */ yytestcase(yyruleno==371); + case 372: /* column_name ::= NK_ID */ yytestcase(yyruleno==372); + case 373: /* function_name ::= NK_ID */ yytestcase(yyruleno==373); + case 374: /* table_alias ::= NK_ID */ yytestcase(yyruleno==374); + case 375: /* column_alias ::= NK_ID */ yytestcase(yyruleno==375); + case 376: /* user_name ::= NK_ID */ yytestcase(yyruleno==376); + case 377: /* topic_name ::= NK_ID */ yytestcase(yyruleno==377); + case 378: /* stream_name ::= NK_ID */ yytestcase(yyruleno==378); + case 379: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==379); + case 380: /* index_name ::= NK_ID */ yytestcase(yyruleno==380); + case 420: /* noarg_func ::= NOW */ yytestcase(yyruleno==420); + case 421: /* noarg_func ::= TODAY */ yytestcase(yyruleno==421); + case 422: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==422); + case 423: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==423); + case 424: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==424); + case 425: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==425); + case 426: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==426); + case 427: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==427); + case 428: /* noarg_func ::= USER */ yytestcase(yyruleno==428); + case 429: /* star_func ::= COUNT */ yytestcase(yyruleno==429); + case 430: /* star_func ::= FIRST */ yytestcase(yyruleno==430); + case 431: /* star_func ::= LAST */ yytestcase(yyruleno==431); + case 432: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==432); { yylhsminor.yy881 = yymsp[0].minor.yy0; } yymsp[0].minor.yy881 = yylhsminor.yy881; break; case 54: /* force_opt ::= */ case 74: /* not_exists_opt ::= */ yytestcase(yyruleno==74); case 76: /* exists_opt ::= */ yytestcase(yyruleno==76); - case 303: /* analyze_opt ::= */ yytestcase(yyruleno==303); - case 310: /* agg_func_opt ::= */ yytestcase(yyruleno==310); - case 494: /* set_quantifier_opt ::= */ yytestcase(yyruleno==494); + case 302: /* analyze_opt ::= */ yytestcase(yyruleno==302); + case 309: /* agg_func_opt ::= */ yytestcase(yyruleno==309); + case 493: /* set_quantifier_opt ::= */ yytestcase(yyruleno==493); { yymsp[1].minor.yy587 = false; } break; case 55: /* force_opt ::= FORCE */ - case 304: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==304); - case 311: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==311); - case 495: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==495); + case 303: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==303); + case 310: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==310); + case 494: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==494); { yymsp[0].minor.yy587 = true; } break; case 56: /* cmd ::= ALTER LOCAL NK_STRING */ @@ -4119,7 +4115,7 @@ static YYACTIONTYPE yy_reduce( yymsp[0].minor.yy220 = yylhsminor.yy220; break; case 122: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 340: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==340); + case 339: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==339); { yylhsminor.yy220 = addNodeToList(pCxt, yymsp[-2].minor.yy220, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } yymsp[-2].minor.yy220 = yylhsminor.yy220; break; @@ -4135,31 +4131,31 @@ static YYACTIONTYPE yy_reduce( case 155: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==155); case 158: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==158); case 165: /* column_def_list ::= column_def */ yytestcase(yyruleno==165); - case 209: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==209); - case 214: /* col_name_list ::= col_name */ yytestcase(yyruleno==214); - case 265: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==265); - case 279: /* func_list ::= func */ yytestcase(yyruleno==279); - case 369: /* literal_list ::= signed_literal */ yytestcase(yyruleno==369); - case 436: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==436); - case 442: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==442); - case 497: /* select_list ::= select_item */ yytestcase(yyruleno==497); - case 508: /* partition_list ::= partition_item */ yytestcase(yyruleno==508); - case 563: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==563); + case 208: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==208); + case 213: /* col_name_list ::= col_name */ yytestcase(yyruleno==213); + case 264: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==264); + case 278: /* func_list ::= func */ yytestcase(yyruleno==278); + case 368: /* literal_list ::= signed_literal */ yytestcase(yyruleno==368); + case 435: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==435); + case 441: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==441); + case 496: /* select_list ::= select_item */ yytestcase(yyruleno==496); + case 507: /* partition_list ::= partition_item */ yytestcase(yyruleno==507); + case 562: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==562); { yylhsminor.yy220 = createNodeList(pCxt, yymsp[0].minor.yy140); } yymsp[0].minor.yy220 = yylhsminor.yy220; break; case 126: /* retention_list ::= retention_list NK_COMMA retention */ case 159: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==159); case 166: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==166); - case 210: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==210); - case 215: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==215); - case 266: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==266); - case 280: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==280); - case 370: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==370); - case 437: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==437); - case 498: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==498); - case 509: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==509); - case 564: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==564); + case 209: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==209); + case 214: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==214); + case 265: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==265); + case 279: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==279); + case 369: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==369); + case 436: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==436); + case 497: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==497); + case 508: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==508); + case 563: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==563); { yylhsminor.yy220 = addNodeToList(pCxt, yymsp[-2].minor.yy220, yymsp[0].minor.yy140); } yymsp[-2].minor.yy220 = yylhsminor.yy220; break; @@ -4168,28 +4164,28 @@ static YYACTIONTYPE yy_reduce( yymsp[-2].minor.yy140 = yylhsminor.yy140; break; case 128: /* speed_opt ::= */ - case 312: /* bufsize_opt ::= */ yytestcase(yyruleno==312); + case 311: /* bufsize_opt ::= */ yytestcase(yyruleno==311); { yymsp[1].minor.yy214 = 0; } break; case 129: /* speed_opt ::= MAX_SPEED NK_INTEGER */ - case 313: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==313); + case 312: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==312); { yymsp[-1].minor.yy214 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 130: /* start_opt ::= */ case 134: /* end_opt ::= */ yytestcase(yyruleno==134); - case 259: /* like_pattern_opt ::= */ yytestcase(yyruleno==259); - case 330: /* subtable_opt ::= */ yytestcase(yyruleno==330); - case 445: /* case_when_else_opt ::= */ yytestcase(yyruleno==445); - case 475: /* from_clause_opt ::= */ yytestcase(yyruleno==475); - case 504: /* where_clause_opt ::= */ yytestcase(yyruleno==504); - case 513: /* twindow_clause_opt ::= */ yytestcase(yyruleno==513); - case 519: /* sliding_opt ::= */ yytestcase(yyruleno==519); - case 521: /* fill_opt ::= */ yytestcase(yyruleno==521); - case 535: /* having_clause_opt ::= */ yytestcase(yyruleno==535); - case 537: /* range_opt ::= */ yytestcase(yyruleno==537); - case 539: /* every_opt ::= */ yytestcase(yyruleno==539); - case 552: /* slimit_clause_opt ::= */ yytestcase(yyruleno==552); - case 556: /* limit_clause_opt ::= */ yytestcase(yyruleno==556); + case 258: /* like_pattern_opt ::= */ yytestcase(yyruleno==258); + case 329: /* subtable_opt ::= */ yytestcase(yyruleno==329); + case 444: /* case_when_else_opt ::= */ yytestcase(yyruleno==444); + case 474: /* from_clause_opt ::= */ yytestcase(yyruleno==474); + case 503: /* where_clause_opt ::= */ yytestcase(yyruleno==503); + case 512: /* twindow_clause_opt ::= */ yytestcase(yyruleno==512); + case 518: /* sliding_opt ::= */ yytestcase(yyruleno==518); + case 520: /* fill_opt ::= */ yytestcase(yyruleno==520); + case 534: /* having_clause_opt ::= */ yytestcase(yyruleno==534); + case 536: /* range_opt ::= */ yytestcase(yyruleno==536); + case 538: /* every_opt ::= */ yytestcase(yyruleno==538); + case 551: /* slimit_clause_opt ::= */ yytestcase(yyruleno==551); + case 555: /* limit_clause_opt ::= */ yytestcase(yyruleno==555); { yymsp[1].minor.yy140 = NULL; } break; case 131: /* start_opt ::= START WITH NK_INTEGER */ @@ -4218,8 +4214,8 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy587, yymsp[0].minor.yy140); } break; case 143: /* cmd ::= ALTER TABLE alter_table_clause */ - case 342: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==342); - case 343: /* cmd ::= insert_query */ yytestcase(yyruleno==343); + case 341: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==341); + case 342: /* cmd ::= insert_query */ yytestcase(yyruleno==342); { pCxt->pRootNode = yymsp[0].minor.yy140; } break; case 144: /* cmd ::= ALTER STABLE alter_table_clause */ @@ -4266,7 +4262,7 @@ static YYACTIONTYPE yy_reduce( yymsp[-5].minor.yy140 = yylhsminor.yy140; break; case 156: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 443: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==443); + case 442: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==442); { yylhsminor.yy220 = addNodeToList(pCxt, yymsp[-1].minor.yy220, yymsp[0].minor.yy140); } yymsp[-1].minor.yy220 = yylhsminor.yy220; break; @@ -4279,17 +4275,17 @@ static YYACTIONTYPE yy_reduce( yymsp[-1].minor.yy140 = yylhsminor.yy140; break; case 161: /* specific_cols_opt ::= */ - case 192: /* tags_def_opt ::= */ yytestcase(yyruleno==192); - case 264: /* tag_list_opt ::= */ yytestcase(yyruleno==264); - case 316: /* col_list_opt ::= */ yytestcase(yyruleno==316); - case 318: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==318); - case 506: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==506); - case 531: /* group_by_clause_opt ::= */ yytestcase(yyruleno==531); - case 550: /* order_by_clause_opt ::= */ yytestcase(yyruleno==550); + case 191: /* tags_def_opt ::= */ yytestcase(yyruleno==191); + case 263: /* tag_list_opt ::= */ yytestcase(yyruleno==263); + case 315: /* col_list_opt ::= */ yytestcase(yyruleno==315); + case 317: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==317); + case 505: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==505); + case 530: /* group_by_clause_opt ::= */ yytestcase(yyruleno==530); + case 549: /* order_by_clause_opt ::= */ yytestcase(yyruleno==549); { yymsp[1].minor.yy220 = NULL; } break; case 162: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ - case 317: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==317); + case 316: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==316); { yymsp[-2].minor.yy220 = yymsp[-1].minor.yy220; } break; case 163: /* full_table_name ::= table_name */ @@ -4304,532 +4300,528 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy140 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy881, yymsp[0].minor.yy682, NULL); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 168: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy140 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy881, yymsp[-2].minor.yy682, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy140 = yylhsminor.yy140; - break; - case 169: /* type_name ::= BOOL */ + case 168: /* type_name ::= BOOL */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 170: /* type_name ::= TINYINT */ + case 169: /* type_name ::= TINYINT */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 171: /* type_name ::= SMALLINT */ + case 170: /* type_name ::= SMALLINT */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 172: /* type_name ::= INT */ - case 173: /* type_name ::= INTEGER */ yytestcase(yyruleno==173); + case 171: /* type_name ::= INT */ + case 172: /* type_name ::= INTEGER */ yytestcase(yyruleno==172); { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 174: /* type_name ::= BIGINT */ + case 173: /* type_name ::= BIGINT */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 175: /* type_name ::= FLOAT */ + case 174: /* type_name ::= FLOAT */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 176: /* type_name ::= DOUBLE */ + case 175: /* type_name ::= DOUBLE */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 177: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + case 176: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ { yymsp[-3].minor.yy682 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 178: /* type_name ::= TIMESTAMP */ + case 177: /* type_name ::= TIMESTAMP */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 179: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + case 178: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ { yymsp[-3].minor.yy682 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 180: /* type_name ::= TINYINT UNSIGNED */ + case 179: /* type_name ::= TINYINT UNSIGNED */ { yymsp[-1].minor.yy682 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 181: /* type_name ::= SMALLINT UNSIGNED */ + case 180: /* type_name ::= SMALLINT UNSIGNED */ { yymsp[-1].minor.yy682 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 182: /* type_name ::= INT UNSIGNED */ + case 181: /* type_name ::= INT UNSIGNED */ { yymsp[-1].minor.yy682 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 183: /* type_name ::= BIGINT UNSIGNED */ + case 182: /* type_name ::= BIGINT UNSIGNED */ { yymsp[-1].minor.yy682 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 184: /* type_name ::= JSON */ + case 183: /* type_name ::= JSON */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 185: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + case 184: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ { yymsp[-3].minor.yy682 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 186: /* type_name ::= MEDIUMBLOB */ + case 185: /* type_name ::= MEDIUMBLOB */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 187: /* type_name ::= BLOB */ + case 186: /* type_name ::= BLOB */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 188: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + case 187: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ { yymsp[-3].minor.yy682 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 189: /* type_name ::= DECIMAL */ + case 188: /* type_name ::= DECIMAL */ { yymsp[0].minor.yy682 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 190: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + case 189: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ { yymsp[-3].minor.yy682 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 191: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + case 190: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ { yymsp[-5].minor.yy682 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 193: /* tags_def_opt ::= tags_def */ - case 319: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==319); - case 435: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==435); + case 192: /* tags_def_opt ::= tags_def */ + case 318: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==318); + case 434: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==434); { yylhsminor.yy220 = yymsp[0].minor.yy220; } yymsp[0].minor.yy220 = yylhsminor.yy220; break; - case 194: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ - case 320: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==320); + case 193: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ + case 319: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==319); { yymsp[-3].minor.yy220 = yymsp[-1].minor.yy220; } break; - case 195: /* table_options ::= */ + case 194: /* table_options ::= */ { yymsp[1].minor.yy140 = createDefaultTableOptions(pCxt); } break; - case 196: /* table_options ::= table_options COMMENT NK_STRING */ + case 195: /* table_options ::= table_options COMMENT NK_STRING */ { yylhsminor.yy140 = setTableOption(pCxt, yymsp[-2].minor.yy140, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 197: /* table_options ::= table_options MAX_DELAY duration_list */ + case 196: /* table_options ::= table_options MAX_DELAY duration_list */ { yylhsminor.yy140 = setTableOption(pCxt, yymsp[-2].minor.yy140, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy220); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 198: /* table_options ::= table_options WATERMARK duration_list */ + case 197: /* table_options ::= table_options WATERMARK duration_list */ { yylhsminor.yy140 = setTableOption(pCxt, yymsp[-2].minor.yy140, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy220); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 199: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + case 198: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ { yylhsminor.yy140 = setTableOption(pCxt, yymsp[-4].minor.yy140, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy220); } yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 200: /* table_options ::= table_options TTL NK_INTEGER */ + case 199: /* table_options ::= table_options TTL NK_INTEGER */ { yylhsminor.yy140 = setTableOption(pCxt, yymsp[-2].minor.yy140, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 201: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + case 200: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ { yylhsminor.yy140 = setTableOption(pCxt, yymsp[-4].minor.yy140, TABLE_OPTION_SMA, yymsp[-1].minor.yy220); } yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 202: /* table_options ::= table_options DELETE_MARK duration_list */ + case 201: /* table_options ::= table_options DELETE_MARK duration_list */ { yylhsminor.yy140 = setTableOption(pCxt, yymsp[-2].minor.yy140, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy220); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 203: /* alter_table_options ::= alter_table_option */ + case 202: /* alter_table_options ::= alter_table_option */ { yylhsminor.yy140 = createAlterTableOptions(pCxt); yylhsminor.yy140 = setTableOption(pCxt, yylhsminor.yy140, yymsp[0].minor.yy809.type, &yymsp[0].minor.yy809.val); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 204: /* alter_table_options ::= alter_table_options alter_table_option */ + case 203: /* alter_table_options ::= alter_table_options alter_table_option */ { yylhsminor.yy140 = setTableOption(pCxt, yymsp[-1].minor.yy140, yymsp[0].minor.yy809.type, &yymsp[0].minor.yy809.val); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 205: /* alter_table_option ::= COMMENT NK_STRING */ + case 204: /* alter_table_option ::= COMMENT NK_STRING */ { yymsp[-1].minor.yy809.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy809.val = yymsp[0].minor.yy0; } break; - case 206: /* alter_table_option ::= TTL NK_INTEGER */ + case 205: /* alter_table_option ::= TTL NK_INTEGER */ { yymsp[-1].minor.yy809.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy809.val = yymsp[0].minor.yy0; } break; - case 207: /* duration_list ::= duration_literal */ - case 399: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==399); + case 206: /* duration_list ::= duration_literal */ + case 398: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==398); { yylhsminor.yy220 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } yymsp[0].minor.yy220 = yylhsminor.yy220; break; - case 208: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 400: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==400); + case 207: /* duration_list ::= duration_list NK_COMMA duration_literal */ + case 399: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==399); { yylhsminor.yy220 = addNodeToList(pCxt, yymsp[-2].minor.yy220, releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } yymsp[-2].minor.yy220 = yylhsminor.yy220; break; - case 211: /* rollup_func_name ::= function_name */ + case 210: /* rollup_func_name ::= function_name */ { yylhsminor.yy140 = createFunctionNode(pCxt, &yymsp[0].minor.yy881, NULL); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 212: /* rollup_func_name ::= FIRST */ - case 213: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==213); - case 268: /* tag_item ::= QTAGS */ yytestcase(yyruleno==268); + case 211: /* rollup_func_name ::= FIRST */ + case 212: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==212); + case 267: /* tag_item ::= QTAGS */ yytestcase(yyruleno==267); { yylhsminor.yy140 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 216: /* col_name ::= column_name */ - case 269: /* tag_item ::= column_name */ yytestcase(yyruleno==269); + case 215: /* col_name ::= column_name */ + case 268: /* tag_item ::= column_name */ yytestcase(yyruleno==268); { yylhsminor.yy140 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy881); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 217: /* cmd ::= SHOW DNODES */ + case 216: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } break; - case 218: /* cmd ::= SHOW USERS */ + case 217: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } break; - case 219: /* cmd ::= SHOW USER PRIVILEGES */ + case 218: /* cmd ::= SHOW USER PRIVILEGES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } break; - case 220: /* cmd ::= SHOW DATABASES */ + case 219: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; - case 221: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + case 220: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy140, yymsp[0].minor.yy140, OP_TYPE_LIKE); } break; - case 222: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + case 221: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy140, yymsp[0].minor.yy140, OP_TYPE_LIKE); } break; - case 223: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ + case 222: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy140, NULL, OP_TYPE_LIKE); } break; - case 224: /* cmd ::= SHOW MNODES */ + case 223: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } break; - case 225: /* cmd ::= SHOW QNODES */ + case 224: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } break; - case 226: /* cmd ::= SHOW FUNCTIONS */ + case 225: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; - case 227: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + case 226: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy140, yymsp[-1].minor.yy140, OP_TYPE_EQUAL); } break; - case 228: /* cmd ::= SHOW STREAMS */ + case 227: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } break; - case 229: /* cmd ::= SHOW ACCOUNTS */ + case 228: /* cmd ::= SHOW ACCOUNTS */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; - case 230: /* cmd ::= SHOW APPS */ + case 229: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } break; - case 231: /* cmd ::= SHOW CONNECTIONS */ + case 230: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } break; - case 232: /* cmd ::= SHOW LICENCES */ - case 233: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==233); + case 231: /* cmd ::= SHOW LICENCES */ + case 232: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==232); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; - case 234: /* cmd ::= SHOW CREATE DATABASE db_name */ + case 233: /* cmd ::= SHOW CREATE DATABASE db_name */ { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy881); } break; - case 235: /* cmd ::= SHOW CREATE TABLE full_table_name */ + case 234: /* cmd ::= SHOW CREATE TABLE full_table_name */ { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy140); } break; - case 236: /* cmd ::= SHOW CREATE STABLE full_table_name */ + case 235: /* cmd ::= SHOW CREATE STABLE full_table_name */ { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy140); } break; - case 237: /* cmd ::= SHOW QUERIES */ + case 236: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } break; - case 238: /* cmd ::= SHOW SCORES */ + case 237: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } break; - case 239: /* cmd ::= SHOW TOPICS */ + case 238: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } break; - case 240: /* cmd ::= SHOW VARIABLES */ - case 241: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==241); + case 239: /* cmd ::= SHOW VARIABLES */ + case 240: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==240); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } break; - case 242: /* cmd ::= SHOW LOCAL VARIABLES */ + case 241: /* cmd ::= SHOW LOCAL VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; - case 243: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + case 242: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ { pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy140); } break; - case 244: /* cmd ::= SHOW BNODES */ + case 243: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } break; - case 245: /* cmd ::= SHOW SNODES */ + case 244: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } break; - case 246: /* cmd ::= SHOW CLUSTER */ + case 245: /* cmd ::= SHOW CLUSTER */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } break; - case 247: /* cmd ::= SHOW TRANSACTIONS */ + case 246: /* cmd ::= SHOW TRANSACTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; - case 248: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + case 247: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ { pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy140); } break; - case 249: /* cmd ::= SHOW CONSUMERS */ + case 248: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } break; - case 250: /* cmd ::= SHOW SUBSCRIPTIONS */ + case 249: /* cmd ::= SHOW SUBSCRIPTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; - case 251: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + case 250: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy140, yymsp[-1].minor.yy140, OP_TYPE_EQUAL); } break; - case 252: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + case 251: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ { pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy140, yymsp[0].minor.yy140, yymsp[-3].minor.yy220); } break; - case 253: /* cmd ::= SHOW VNODES NK_INTEGER */ + case 252: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } break; - case 254: /* cmd ::= SHOW VNODES NK_STRING */ + case 253: /* cmd ::= SHOW VNODES NK_STRING */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } break; - case 255: /* cmd ::= SHOW db_name_cond_opt ALIVE */ + case 254: /* cmd ::= SHOW db_name_cond_opt ALIVE */ { pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy140, QUERY_NODE_SHOW_DB_ALIVE_STMT); } break; - case 256: /* cmd ::= SHOW CLUSTER ALIVE */ + case 255: /* cmd ::= SHOW CLUSTER ALIVE */ { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } break; - case 257: /* db_name_cond_opt ::= */ - case 262: /* from_db_opt ::= */ yytestcase(yyruleno==262); + case 256: /* db_name_cond_opt ::= */ + case 261: /* from_db_opt ::= */ yytestcase(yyruleno==261); { yymsp[1].minor.yy140 = createDefaultDatabaseCondValue(pCxt); } break; - case 258: /* db_name_cond_opt ::= db_name NK_DOT */ + case 257: /* db_name_cond_opt ::= db_name NK_DOT */ { yylhsminor.yy140 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy881); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 260: /* like_pattern_opt ::= LIKE NK_STRING */ + case 259: /* like_pattern_opt ::= LIKE NK_STRING */ { yymsp[-1].minor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 261: /* table_name_cond ::= table_name */ + case 260: /* table_name_cond ::= table_name */ { yylhsminor.yy140 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy881); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 263: /* from_db_opt ::= FROM db_name */ + case 262: /* from_db_opt ::= FROM db_name */ { yymsp[-1].minor.yy140 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy881); } break; - case 267: /* tag_item ::= TBNAME */ + case 266: /* tag_item ::= TBNAME */ { yylhsminor.yy140 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 270: /* tag_item ::= column_name column_alias */ + case 269: /* tag_item ::= column_name column_alias */ { yylhsminor.yy140 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy881), &yymsp[0].minor.yy881); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 271: /* tag_item ::= column_name AS column_alias */ + case 270: /* tag_item ::= column_name AS column_alias */ { yylhsminor.yy140 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy881), &yymsp[0].minor.yy881); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 272: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + case 271: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy587, yymsp[-3].minor.yy140, yymsp[-1].minor.yy140, NULL, yymsp[0].minor.yy140); } break; - case 273: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + case 272: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy587, yymsp[-5].minor.yy140, yymsp[-3].minor.yy140, yymsp[-1].minor.yy220, NULL); } break; - case 274: /* cmd ::= DROP INDEX exists_opt full_index_name */ + case 273: /* cmd ::= DROP INDEX exists_opt full_index_name */ { pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy587, yymsp[0].minor.yy140); } break; - case 275: /* full_index_name ::= index_name */ + case 274: /* full_index_name ::= index_name */ { yylhsminor.yy140 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy881); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 276: /* full_index_name ::= db_name NK_DOT index_name */ + case 275: /* full_index_name ::= db_name NK_DOT index_name */ { yylhsminor.yy140 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy881, &yymsp[0].minor.yy881); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 277: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + case 276: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ { yymsp[-9].minor.yy140 = createIndexOption(pCxt, yymsp[-7].minor.yy220, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), NULL, yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } break; - case 278: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + case 277: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ { yymsp[-11].minor.yy140 = createIndexOption(pCxt, yymsp[-9].minor.yy220, releaseRawExprNode(pCxt, yymsp[-5].minor.yy140), releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } break; - case 281: /* func ::= sma_func_name NK_LP expression_list NK_RP */ + case 280: /* func ::= sma_func_name NK_LP expression_list NK_RP */ { yylhsminor.yy140 = createFunctionNode(pCxt, &yymsp[-3].minor.yy881, yymsp[-1].minor.yy220); } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 287: /* sma_stream_opt ::= */ - case 321: /* stream_options ::= */ yytestcase(yyruleno==321); + case 286: /* sma_stream_opt ::= */ + case 320: /* stream_options ::= */ yytestcase(yyruleno==320); { yymsp[1].minor.yy140 = createStreamOptions(pCxt); } break; - case 288: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + case 287: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ { ((SStreamOptions*)yymsp[-2].minor.yy140)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy140); yylhsminor.yy140 = yymsp[-2].minor.yy140; } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 289: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + case 288: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ { ((SStreamOptions*)yymsp[-2].minor.yy140)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy140); yylhsminor.yy140 = yymsp[-2].minor.yy140; } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 290: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + case 289: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ { ((SStreamOptions*)yymsp[-2].minor.yy140)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy140); yylhsminor.yy140 = yymsp[-2].minor.yy140; } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 291: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + case 290: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy587, &yymsp[-2].minor.yy881, yymsp[0].minor.yy140); } break; - case 292: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ + case 291: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy587, &yymsp[-3].minor.yy881, &yymsp[0].minor.yy881, false); } break; - case 293: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ + case 292: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy587, &yymsp[-5].minor.yy881, &yymsp[0].minor.yy881, true); } break; - case 294: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ + case 293: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy587, &yymsp[-3].minor.yy881, yymsp[0].minor.yy140, false); } break; - case 295: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + case 294: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy587, &yymsp[-5].minor.yy881, yymsp[0].minor.yy140, true); } break; - case 296: /* cmd ::= DROP TOPIC exists_opt topic_name */ + case 295: /* cmd ::= DROP TOPIC exists_opt topic_name */ { pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy587, &yymsp[0].minor.yy881); } break; - case 297: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + case 296: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ { pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy587, &yymsp[-2].minor.yy881, &yymsp[0].minor.yy881); } break; - case 298: /* cmd ::= DESC full_table_name */ - case 299: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==299); + case 297: /* cmd ::= DESC full_table_name */ + case 298: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==298); { pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy140); } break; - case 300: /* cmd ::= RESET QUERY CACHE */ + case 299: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 301: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - case 302: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==302); + case 300: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + case 301: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==301); { pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy587, yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } break; - case 305: /* explain_options ::= */ + case 304: /* explain_options ::= */ { yymsp[1].minor.yy140 = createDefaultExplainOptions(pCxt); } break; - case 306: /* explain_options ::= explain_options VERBOSE NK_BOOL */ + case 305: /* explain_options ::= explain_options VERBOSE NK_BOOL */ { yylhsminor.yy140 = setExplainVerbose(pCxt, yymsp[-2].minor.yy140, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 307: /* explain_options ::= explain_options RATIO NK_FLOAT */ + case 306: /* explain_options ::= explain_options RATIO NK_FLOAT */ { yylhsminor.yy140 = setExplainRatio(pCxt, yymsp[-2].minor.yy140, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 308: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ + case 307: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ { pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy587, yymsp[-8].minor.yy587, &yymsp[-5].minor.yy881, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy682, yymsp[0].minor.yy214); } break; - case 309: /* cmd ::= DROP FUNCTION exists_opt function_name */ + case 308: /* cmd ::= DROP FUNCTION exists_opt function_name */ { pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy587, &yymsp[0].minor.yy881); } break; - case 314: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ + case 313: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ { pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy587, &yymsp[-8].minor.yy881, yymsp[-5].minor.yy140, yymsp[-7].minor.yy140, yymsp[-3].minor.yy220, yymsp[-2].minor.yy140, yymsp[0].minor.yy140, yymsp[-4].minor.yy220); } break; - case 315: /* cmd ::= DROP STREAM exists_opt stream_name */ + case 314: /* cmd ::= DROP STREAM exists_opt stream_name */ { pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy587, &yymsp[0].minor.yy881); } break; - case 322: /* stream_options ::= stream_options TRIGGER AT_ONCE */ - case 323: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==323); + case 321: /* stream_options ::= stream_options TRIGGER AT_ONCE */ + case 322: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==322); { yylhsminor.yy140 = setStreamOptions(pCxt, yymsp[-2].minor.yy140, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 324: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + case 323: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ { yylhsminor.yy140 = setStreamOptions(pCxt, yymsp[-3].minor.yy140, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 325: /* stream_options ::= stream_options WATERMARK duration_literal */ + case 324: /* stream_options ::= stream_options WATERMARK duration_literal */ { yylhsminor.yy140 = setStreamOptions(pCxt, yymsp[-2].minor.yy140, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 326: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + case 325: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ { yylhsminor.yy140 = setStreamOptions(pCxt, yymsp[-3].minor.yy140, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 327: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + case 326: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ { yylhsminor.yy140 = setStreamOptions(pCxt, yymsp[-2].minor.yy140, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 328: /* stream_options ::= stream_options DELETE_MARK duration_literal */ + case 327: /* stream_options ::= stream_options DELETE_MARK duration_literal */ { yylhsminor.yy140 = setStreamOptions(pCxt, yymsp[-2].minor.yy140, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 329: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + case 328: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ { yylhsminor.yy140 = setStreamOptions(pCxt, yymsp[-3].minor.yy140, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 331: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 520: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==520); - case 540: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==540); + case 330: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + case 519: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==519); + case 539: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==539); { yymsp[-3].minor.yy140 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy140); } break; - case 332: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 331: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 333: /* cmd ::= KILL QUERY NK_STRING */ + case 332: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 334: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 333: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 335: /* cmd ::= BALANCE VGROUP */ + case 334: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 336: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 335: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 337: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + case 336: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy220); } break; - case 338: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 337: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 339: /* dnode_list ::= DNODE NK_INTEGER */ + case 338: /* dnode_list ::= DNODE NK_INTEGER */ { yymsp[-1].minor.yy220 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } break; - case 341: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ + case 340: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ { pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } break; - case 344: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + case 343: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ { yymsp[-6].minor.yy140 = createInsertStmt(pCxt, yymsp[-4].minor.yy140, yymsp[-2].minor.yy220, yymsp[0].minor.yy140); } break; - case 345: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ + case 344: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ { yymsp[-3].minor.yy140 = createInsertStmt(pCxt, yymsp[-1].minor.yy140, NULL, yymsp[0].minor.yy140); } break; - case 346: /* literal ::= NK_INTEGER */ + case 345: /* literal ::= NK_INTEGER */ { yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 347: /* literal ::= NK_FLOAT */ + case 346: /* literal ::= NK_FLOAT */ { yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 348: /* literal ::= NK_STRING */ + case 347: /* literal ::= NK_STRING */ { yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 349: /* literal ::= NK_BOOL */ + case 348: /* literal ::= NK_BOOL */ { yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 350: /* literal ::= TIMESTAMP NK_STRING */ + case 349: /* literal ::= TIMESTAMP NK_STRING */ { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 351: /* literal ::= duration_literal */ - case 361: /* signed_literal ::= signed */ yytestcase(yyruleno==361); - case 382: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==382); - case 383: /* expression ::= literal */ yytestcase(yyruleno==383); - case 384: /* expression ::= pseudo_column */ yytestcase(yyruleno==384); - case 385: /* expression ::= column_reference */ yytestcase(yyruleno==385); - case 386: /* expression ::= function_expression */ yytestcase(yyruleno==386); - case 387: /* expression ::= case_when_expression */ yytestcase(yyruleno==387); - case 418: /* function_expression ::= literal_func */ yytestcase(yyruleno==418); - case 467: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==467); - case 471: /* boolean_primary ::= predicate */ yytestcase(yyruleno==471); - case 473: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==473); - case 474: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==474); - case 477: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==477); - case 479: /* table_reference ::= table_primary */ yytestcase(yyruleno==479); - case 480: /* table_reference ::= joined_table */ yytestcase(yyruleno==480); - case 484: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==484); - case 542: /* query_simple ::= query_specification */ yytestcase(yyruleno==542); - case 543: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==543); - case 546: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==546); - case 548: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==548); + case 350: /* literal ::= duration_literal */ + case 360: /* signed_literal ::= signed */ yytestcase(yyruleno==360); + case 381: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==381); + case 382: /* expression ::= literal */ yytestcase(yyruleno==382); + case 383: /* expression ::= pseudo_column */ yytestcase(yyruleno==383); + case 384: /* expression ::= column_reference */ yytestcase(yyruleno==384); + case 385: /* expression ::= function_expression */ yytestcase(yyruleno==385); + case 386: /* expression ::= case_when_expression */ yytestcase(yyruleno==386); + case 417: /* function_expression ::= literal_func */ yytestcase(yyruleno==417); + case 466: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==466); + case 470: /* boolean_primary ::= predicate */ yytestcase(yyruleno==470); + case 472: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==472); + case 473: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==473); + case 476: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==476); + case 478: /* table_reference ::= table_primary */ yytestcase(yyruleno==478); + case 479: /* table_reference ::= joined_table */ yytestcase(yyruleno==479); + case 483: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==483); + case 541: /* query_simple ::= query_specification */ yytestcase(yyruleno==541); + case 542: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==542); + case 545: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==545); + case 547: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==547); { yylhsminor.yy140 = yymsp[0].minor.yy140; } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 352: /* literal ::= NULL */ + case 351: /* literal ::= NULL */ { yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 353: /* literal ::= NK_QUESTION */ + case 352: /* literal ::= NK_QUESTION */ { yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 354: /* duration_literal ::= NK_VARIABLE */ + case 353: /* duration_literal ::= NK_VARIABLE */ { yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 355: /* signed ::= NK_INTEGER */ + case 354: /* signed ::= NK_INTEGER */ { yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 356: /* signed ::= NK_PLUS NK_INTEGER */ + case 355: /* signed ::= NK_PLUS NK_INTEGER */ { yymsp[-1].minor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } break; - case 357: /* signed ::= NK_MINUS NK_INTEGER */ + case 356: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; @@ -4837,14 +4829,14 @@ static YYACTIONTYPE yy_reduce( } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 358: /* signed ::= NK_FLOAT */ + case 357: /* signed ::= NK_FLOAT */ { yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 359: /* signed ::= NK_PLUS NK_FLOAT */ + case 358: /* signed ::= NK_PLUS NK_FLOAT */ { yymsp[-1].minor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 360: /* signed ::= NK_MINUS NK_FLOAT */ + case 359: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; @@ -4852,57 +4844,57 @@ static YYACTIONTYPE yy_reduce( } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 362: /* signed_literal ::= NK_STRING */ + case 361: /* signed_literal ::= NK_STRING */ { yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 363: /* signed_literal ::= NK_BOOL */ + case 362: /* signed_literal ::= NK_BOOL */ { yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 364: /* signed_literal ::= TIMESTAMP NK_STRING */ + case 363: /* signed_literal ::= TIMESTAMP NK_STRING */ { yymsp[-1].minor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; - case 365: /* signed_literal ::= duration_literal */ - case 367: /* signed_literal ::= literal_func */ yytestcase(yyruleno==367); - case 438: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==438); - case 500: /* select_item ::= common_expression */ yytestcase(yyruleno==500); - case 510: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==510); - case 547: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==547); - case 549: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==549); - case 562: /* search_condition ::= common_expression */ yytestcase(yyruleno==562); + case 364: /* signed_literal ::= duration_literal */ + case 366: /* signed_literal ::= literal_func */ yytestcase(yyruleno==366); + case 437: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==437); + case 499: /* select_item ::= common_expression */ yytestcase(yyruleno==499); + case 509: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==509); + case 546: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==546); + case 548: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==548); + case 561: /* search_condition ::= common_expression */ yytestcase(yyruleno==561); { yylhsminor.yy140 = releaseRawExprNode(pCxt, yymsp[0].minor.yy140); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 366: /* signed_literal ::= NULL */ + case 365: /* signed_literal ::= NULL */ { yylhsminor.yy140 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 368: /* signed_literal ::= NK_QUESTION */ + case 367: /* signed_literal ::= NK_QUESTION */ { yylhsminor.yy140 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 388: /* expression ::= NK_LP expression NK_RP */ - case 472: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==472); - case 561: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==561); + case 387: /* expression ::= NK_LP expression NK_RP */ + case 471: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==471); + case 560: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==560); { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy140)); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 389: /* expression ::= NK_PLUS expr_or_subquery */ + case 388: /* expression ::= NK_PLUS expr_or_subquery */ { SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 390: /* expression ::= NK_MINUS expr_or_subquery */ + case 389: /* expression ::= NK_MINUS expr_or_subquery */ { SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy140), NULL)); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 391: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + case 390: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -4910,7 +4902,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 392: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + case 391: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -4918,7 +4910,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 393: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + case 392: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -4926,7 +4918,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 394: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + case 393: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -4934,7 +4926,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 395: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ + case 394: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -4942,14 +4934,14 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 396: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 395: /* expression ::= column_reference NK_ARROW NK_STRING */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 397: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + case 396: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -4957,7 +4949,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 398: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + case 397: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -4965,71 +4957,71 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 401: /* column_reference ::= column_name */ + case 400: /* column_reference ::= column_name */ { yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy881, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy881)); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 402: /* column_reference ::= table_name NK_DOT column_name */ + case 401: /* column_reference ::= table_name NK_DOT column_name */ { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy881, &yymsp[0].minor.yy881, createColumnNode(pCxt, &yymsp[-2].minor.yy881, &yymsp[0].minor.yy881)); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 403: /* pseudo_column ::= ROWTS */ - case 404: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==404); - case 406: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==406); - case 407: /* pseudo_column ::= QEND */ yytestcase(yyruleno==407); - case 408: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==408); - case 409: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==409); - case 410: /* pseudo_column ::= WEND */ yytestcase(yyruleno==410); - case 411: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==411); - case 412: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==412); - case 413: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==413); - case 414: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==414); - case 420: /* literal_func ::= NOW */ yytestcase(yyruleno==420); + case 402: /* pseudo_column ::= ROWTS */ + case 403: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==403); + case 405: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==405); + case 406: /* pseudo_column ::= QEND */ yytestcase(yyruleno==406); + case 407: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==407); + case 408: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==408); + case 409: /* pseudo_column ::= WEND */ yytestcase(yyruleno==409); + case 410: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==410); + case 411: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==411); + case 412: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==412); + case 413: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==413); + case 419: /* literal_func ::= NOW */ yytestcase(yyruleno==419); { yylhsminor.yy140 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 405: /* pseudo_column ::= table_name NK_DOT TBNAME */ + case 404: /* pseudo_column ::= table_name NK_DOT TBNAME */ { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy881, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy881)))); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 415: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 416: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==416); + case 414: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 415: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==415); { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy881, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy881, yymsp[-1].minor.yy220)); } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 417: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + case 416: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), yymsp[-1].minor.yy682)); } yymsp[-5].minor.yy140 = yylhsminor.yy140; break; - case 419: /* literal_func ::= noarg_func NK_LP NK_RP */ + case 418: /* literal_func ::= noarg_func NK_LP NK_RP */ { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy881, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy881, NULL)); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 434: /* star_func_para_list ::= NK_STAR */ + case 433: /* star_func_para_list ::= NK_STAR */ { yylhsminor.yy220 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } yymsp[0].minor.yy220 = yylhsminor.yy220; break; - case 439: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 503: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==503); + case 438: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 502: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==502); { yylhsminor.yy140 = createColumnNode(pCxt, &yymsp[-2].minor.yy881, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 440: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ + case 439: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy220, yymsp[-1].minor.yy140)); } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 441: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + case 440: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), yymsp[-2].minor.yy220, yymsp[-1].minor.yy140)); } yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 444: /* when_then_expr ::= WHEN common_expression THEN common_expression */ + case 443: /* when_then_expr ::= WHEN common_expression THEN common_expression */ { yymsp[-3].minor.yy140 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), releaseRawExprNode(pCxt, yymsp[0].minor.yy140)); } break; - case 446: /* case_when_else_opt ::= ELSE common_expression */ + case 445: /* case_when_else_opt ::= ELSE common_expression */ { yymsp[-1].minor.yy140 = releaseRawExprNode(pCxt, yymsp[0].minor.yy140); } break; - case 447: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ - case 452: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==452); + case 446: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ + case 451: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==451); { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -5037,7 +5029,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 448: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + case 447: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -5045,7 +5037,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-4].minor.yy140 = yylhsminor.yy140; break; - case 449: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + case 448: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -5053,71 +5045,71 @@ static YYACTIONTYPE yy_reduce( } yymsp[-5].minor.yy140 = yylhsminor.yy140; break; - case 450: /* predicate ::= expr_or_subquery IS NULL */ + case 449: /* predicate ::= expr_or_subquery IS NULL */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), NULL)); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 451: /* predicate ::= expr_or_subquery IS NOT NULL */ + case 450: /* predicate ::= expr_or_subquery IS NOT NULL */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy140); yylhsminor.yy140 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), NULL)); } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 453: /* compare_op ::= NK_LT */ + case 452: /* compare_op ::= NK_LT */ { yymsp[0].minor.yy794 = OP_TYPE_LOWER_THAN; } break; - case 454: /* compare_op ::= NK_GT */ + case 453: /* compare_op ::= NK_GT */ { yymsp[0].minor.yy794 = OP_TYPE_GREATER_THAN; } break; - case 455: /* compare_op ::= NK_LE */ + case 454: /* compare_op ::= NK_LE */ { yymsp[0].minor.yy794 = OP_TYPE_LOWER_EQUAL; } break; - case 456: /* compare_op ::= NK_GE */ + case 455: /* compare_op ::= NK_GE */ { yymsp[0].minor.yy794 = OP_TYPE_GREATER_EQUAL; } break; - case 457: /* compare_op ::= NK_NE */ + case 456: /* compare_op ::= NK_NE */ { yymsp[0].minor.yy794 = OP_TYPE_NOT_EQUAL; } break; - case 458: /* compare_op ::= NK_EQ */ + case 457: /* compare_op ::= NK_EQ */ { yymsp[0].minor.yy794 = OP_TYPE_EQUAL; } break; - case 459: /* compare_op ::= LIKE */ + case 458: /* compare_op ::= LIKE */ { yymsp[0].minor.yy794 = OP_TYPE_LIKE; } break; - case 460: /* compare_op ::= NOT LIKE */ + case 459: /* compare_op ::= NOT LIKE */ { yymsp[-1].minor.yy794 = OP_TYPE_NOT_LIKE; } break; - case 461: /* compare_op ::= MATCH */ + case 460: /* compare_op ::= MATCH */ { yymsp[0].minor.yy794 = OP_TYPE_MATCH; } break; - case 462: /* compare_op ::= NMATCH */ + case 461: /* compare_op ::= NMATCH */ { yymsp[0].minor.yy794 = OP_TYPE_NMATCH; } break; - case 463: /* compare_op ::= CONTAINS */ + case 462: /* compare_op ::= CONTAINS */ { yymsp[0].minor.yy794 = OP_TYPE_JSON_CONTAINS; } break; - case 464: /* in_op ::= IN */ + case 463: /* in_op ::= IN */ { yymsp[0].minor.yy794 = OP_TYPE_IN; } break; - case 465: /* in_op ::= NOT IN */ + case 464: /* in_op ::= NOT IN */ { yymsp[-1].minor.yy794 = OP_TYPE_NOT_IN; } break; - case 466: /* in_predicate_value ::= NK_LP literal_list NK_RP */ + case 465: /* in_predicate_value ::= NK_LP literal_list NK_RP */ { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy220)); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 468: /* boolean_value_expression ::= NOT boolean_primary */ + case 467: /* boolean_value_expression ::= NOT boolean_primary */ { SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy140), NULL)); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 469: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 468: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -5125,7 +5117,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 470: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 469: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy140); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy140); @@ -5133,48 +5125,48 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 476: /* from_clause_opt ::= FROM table_reference_list */ - case 505: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==505); - case 536: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==536); + case 475: /* from_clause_opt ::= FROM table_reference_list */ + case 504: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==504); + case 535: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==535); { yymsp[-1].minor.yy140 = yymsp[0].minor.yy140; } break; - case 478: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ + case 477: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ { yylhsminor.yy140 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy140, yymsp[0].minor.yy140, NULL); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 481: /* table_primary ::= table_name alias_opt */ + case 480: /* table_primary ::= table_name alias_opt */ { yylhsminor.yy140 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy881, &yymsp[0].minor.yy881); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 482: /* table_primary ::= db_name NK_DOT table_name alias_opt */ + case 481: /* table_primary ::= db_name NK_DOT table_name alias_opt */ { yylhsminor.yy140 = createRealTableNode(pCxt, &yymsp[-3].minor.yy881, &yymsp[-1].minor.yy881, &yymsp[0].minor.yy881); } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 483: /* table_primary ::= subquery alias_opt */ + case 482: /* table_primary ::= subquery alias_opt */ { yylhsminor.yy140 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy140), &yymsp[0].minor.yy881); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 485: /* alias_opt ::= */ + case 484: /* alias_opt ::= */ { yymsp[1].minor.yy881 = nil_token; } break; - case 487: /* alias_opt ::= AS table_alias */ + case 486: /* alias_opt ::= AS table_alias */ { yymsp[-1].minor.yy881 = yymsp[0].minor.yy881; } break; - case 488: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 489: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==489); + case 487: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 488: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==488); { yymsp[-2].minor.yy140 = yymsp[-1].minor.yy140; } break; - case 490: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + case 489: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ { yylhsminor.yy140 = createJoinTableNode(pCxt, yymsp[-4].minor.yy852, yymsp[-5].minor.yy140, yymsp[-2].minor.yy140, yymsp[0].minor.yy140); } yymsp[-5].minor.yy140 = yylhsminor.yy140; break; - case 491: /* join_type ::= */ + case 490: /* join_type ::= */ { yymsp[1].minor.yy852 = JOIN_TYPE_INNER; } break; - case 492: /* join_type ::= INNER */ + case 491: /* join_type ::= INNER */ { yymsp[0].minor.yy852 = JOIN_TYPE_INNER; } break; - case 493: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 492: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { yymsp[-11].minor.yy140 = createSelectStmt(pCxt, yymsp[-10].minor.yy587, yymsp[-9].minor.yy220, yymsp[-8].minor.yy140); yymsp[-11].minor.yy140 = addWhereClause(pCxt, yymsp[-11].minor.yy140, yymsp[-7].minor.yy140); @@ -5187,82 +5179,82 @@ static YYACTIONTYPE yy_reduce( yymsp[-11].minor.yy140 = addFillClause(pCxt, yymsp[-11].minor.yy140, yymsp[-3].minor.yy140); } break; - case 496: /* set_quantifier_opt ::= ALL */ + case 495: /* set_quantifier_opt ::= ALL */ { yymsp[0].minor.yy587 = false; } break; - case 499: /* select_item ::= NK_STAR */ + case 498: /* select_item ::= NK_STAR */ { yylhsminor.yy140 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } yymsp[0].minor.yy140 = yylhsminor.yy140; break; - case 501: /* select_item ::= common_expression column_alias */ - case 511: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==511); + case 500: /* select_item ::= common_expression column_alias */ + case 510: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==510); { yylhsminor.yy140 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy140), &yymsp[0].minor.yy881); } yymsp[-1].minor.yy140 = yylhsminor.yy140; break; - case 502: /* select_item ::= common_expression AS column_alias */ - case 512: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==512); + case 501: /* select_item ::= common_expression AS column_alias */ + case 511: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==511); { yylhsminor.yy140 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), &yymsp[0].minor.yy881); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 507: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 532: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==532); - case 551: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==551); + case 506: /* partition_by_clause_opt ::= PARTITION BY partition_list */ + case 531: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==531); + case 550: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==550); { yymsp[-2].minor.yy220 = yymsp[0].minor.yy220; } break; - case 514: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + case 513: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ { yymsp[-5].minor.yy140 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), releaseRawExprNode(pCxt, yymsp[-1].minor.yy140)); } break; - case 515: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + case 514: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ { yymsp[-3].minor.yy140 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy140)); } break; - case 516: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + case 515: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ { yymsp[-5].minor.yy140 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), NULL, yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } break; - case 517: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + case 516: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ { yymsp[-7].minor.yy140 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy140), releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), yymsp[-1].minor.yy140, yymsp[0].minor.yy140); } break; - case 518: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + case 517: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ { yymsp[-6].minor.yy140 = createEventWindowNode(pCxt, yymsp[-3].minor.yy140, yymsp[0].minor.yy140); } break; - case 522: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ + case 521: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ { yymsp[-3].minor.yy140 = createFillNode(pCxt, yymsp[-1].minor.yy174, NULL); } break; - case 523: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + case 522: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ { yymsp[-5].minor.yy140 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy220)); } break; - case 524: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ + case 523: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ { yymsp[-5].minor.yy140 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy220)); } break; - case 525: /* fill_mode ::= NONE */ + case 524: /* fill_mode ::= NONE */ { yymsp[0].minor.yy174 = FILL_MODE_NONE; } break; - case 526: /* fill_mode ::= PREV */ + case 525: /* fill_mode ::= PREV */ { yymsp[0].minor.yy174 = FILL_MODE_PREV; } break; - case 527: /* fill_mode ::= NULL */ + case 526: /* fill_mode ::= NULL */ { yymsp[0].minor.yy174 = FILL_MODE_NULL; } break; - case 528: /* fill_mode ::= NULL_F */ + case 527: /* fill_mode ::= NULL_F */ { yymsp[0].minor.yy174 = FILL_MODE_NULL_F; } break; - case 529: /* fill_mode ::= LINEAR */ + case 528: /* fill_mode ::= LINEAR */ { yymsp[0].minor.yy174 = FILL_MODE_LINEAR; } break; - case 530: /* fill_mode ::= NEXT */ + case 529: /* fill_mode ::= NEXT */ { yymsp[0].minor.yy174 = FILL_MODE_NEXT; } break; - case 533: /* group_by_list ::= expr_or_subquery */ + case 532: /* group_by_list ::= expr_or_subquery */ { yylhsminor.yy220 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } yymsp[0].minor.yy220 = yylhsminor.yy220; break; - case 534: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + case 533: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ { yylhsminor.yy220 = addNodeToList(pCxt, yymsp[-2].minor.yy220, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy140))); } yymsp[-2].minor.yy220 = yylhsminor.yy220; break; - case 538: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + case 537: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ { yymsp[-5].minor.yy140 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy140), releaseRawExprNode(pCxt, yymsp[-1].minor.yy140)); } break; - case 541: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 540: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { yylhsminor.yy140 = addOrderByClause(pCxt, yymsp[-3].minor.yy140, yymsp[-2].minor.yy220); yylhsminor.yy140 = addSlimitClause(pCxt, yylhsminor.yy140, yymsp[-1].minor.yy140); @@ -5270,50 +5262,50 @@ static YYACTIONTYPE yy_reduce( } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 544: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + case 543: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ { yylhsminor.yy140 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy140, yymsp[0].minor.yy140); } yymsp[-3].minor.yy140 = yylhsminor.yy140; break; - case 545: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + case 544: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ { yylhsminor.yy140 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy140, yymsp[0].minor.yy140); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 553: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 557: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==557); + case 552: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 556: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==556); { yymsp[-1].minor.yy140 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 554: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 558: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==558); + case 553: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 557: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==557); { yymsp[-3].minor.yy140 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 555: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 559: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==559); + case 554: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 558: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==558); { yymsp[-3].minor.yy140 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 560: /* subquery ::= NK_LP query_expression NK_RP */ + case 559: /* subquery ::= NK_LP query_expression NK_RP */ { yylhsminor.yy140 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy140); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 565: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + case 564: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ { yylhsminor.yy140 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy140), yymsp[-1].minor.yy866, yymsp[0].minor.yy697); } yymsp[-2].minor.yy140 = yylhsminor.yy140; break; - case 566: /* ordering_specification_opt ::= */ + case 565: /* ordering_specification_opt ::= */ { yymsp[1].minor.yy866 = ORDER_ASC; } break; - case 567: /* ordering_specification_opt ::= ASC */ + case 566: /* ordering_specification_opt ::= ASC */ { yymsp[0].minor.yy866 = ORDER_ASC; } break; - case 568: /* ordering_specification_opt ::= DESC */ + case 567: /* ordering_specification_opt ::= DESC */ { yymsp[0].minor.yy866 = ORDER_DESC; } break; - case 569: /* null_ordering_opt ::= */ + case 568: /* null_ordering_opt ::= */ { yymsp[1].minor.yy697 = NULL_ORDER_DEFAULT; } break; - case 570: /* null_ordering_opt ::= NULLS FIRST */ + case 569: /* null_ordering_opt ::= NULLS FIRST */ { yymsp[-1].minor.yy697 = NULL_ORDER_FIRST; } break; - case 571: /* null_ordering_opt ::= NULLS LAST */ + case 570: /* null_ordering_opt ::= NULLS LAST */ { yymsp[-1].minor.yy697 = NULL_ORDER_LAST; } break; default: diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index c36402e8e0..6981f79914 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -742,10 +742,10 @@ TEST_F(ParserInitialCTest, createStable) { addFieldToCreateStbReq(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); run("CREATE STABLE IF NOT EXISTS rollup_db.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " - "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c14 VARCHAR(50)) " "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, " - "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, " + "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, " "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m " "DELETE_MARK 1000s,200m"); @@ -1005,16 +1005,16 @@ TEST_F(ParserInitialCTest, createTable) { run("CREATE TABLE IF NOT EXISTS test.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " - "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)"); run("CREATE TABLE IF NOT EXISTS rollup_db.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " - "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c14 VARCHAR(50)) " "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, a8 BINARY(20), " - "a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, " + "a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, " "a14 NCHAR(30), a15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)"); From 519e2f1cfd5b4f23e05d0c982b23a39f27472f8a Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 4 Apr 2023 18:21:37 +0800 Subject: [PATCH 150/176] fix:tq memory leak --- source/dnode/vnode/src/tq/tq.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d4bdd633e9..dcfbf7365a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1559,6 +1559,8 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); return 0; + } else { + tDeleteStreamDispatchReq(&req); } code = TSDB_CODE_STREAM_TASK_NOT_EXIST; From 949193e9e7236322a0a833588f6bbefa419f571e Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Wed, 5 Apr 2023 00:53:04 +0800 Subject: [PATCH 151/176] release: upgrade default version to 3.0.3.2 (#20766) --- cmake/cmake.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 42de285c8e..5150ee3b75 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.3.1") + SET(TD_VER_NUMBER "3.0.3.2") ENDIF () IF (DEFINED VERCOMPATIBLE) From 13991b61060184709d65ec4791ce00340eafaff9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 5 Apr 2023 12:18:01 +0800 Subject: [PATCH 152/176] fix: taosbenchmark disorder dynamic range for main (#20774) * fix: taosbenchmark disorder dynamic range for main * fix: merge with main --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 2de3881dd2..3f27cb2517 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 273a3fe + GIT_TAG ddd654a SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From d7de7d5aeae6f8572ec4d0591bc75d14a2bba21a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 6 Apr 2023 12:21:16 +0800 Subject: [PATCH 153/176] fix: taosbenchmark child from/to for main (#20780) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 3f27cb2517..a9e6b57000 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG ddd654a + GIT_TAG 6ae60ac SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 18955be092bc5d62cf8d6f03661e3d34ac6dc018 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Apr 2023 14:40:01 +0800 Subject: [PATCH 154/176] refactor: erase unnecessary error log. --- source/libs/executor/src/executor.c | 11 +++-------- source/libs/executor/src/executorimpl.c | 13 ++++++++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 5a7ff42ddf..64fb2e80e9 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -503,12 +503,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, if (handle) { void* pSinkParam = NULL; - - SArray* pInfoList = getTableListInfo(*pTask); - STableListInfo* pTableListInfo = taosArrayGetP(pInfoList, 0); - taosArrayDestroy(pInfoList); - - code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTableListInfo, readHandle); + code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, (*pTask), readHandle); if (code != TSDB_CODE_SUCCESS) { qError("failed to createDataSinkParam, vgId:%d, code:%s, %s", vgId, tstrerror(code), (*pTask)->id.str); goto _error; @@ -1100,14 +1095,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SStreamScanInfo* pInfo = pOperator->info; STableScanInfo* pScanInfo = pInfo->pTableScanOp->info; STableScanBase* pScanBaseInfo = &pScanInfo->base; - STableListInfo* pTableListInfo = pScanBaseInfo->pTableListInfo; + STableListInfo* pTableListInfo = pScanBaseInfo->pTableListInfo; if (pOffset->type == TMQ_OFFSET__LOG) { tsdbReaderClose(pScanBaseInfo->dataReader); pScanBaseInfo->dataReader = NULL; // let's seek to the next version in wal file - if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) { + if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { qError("tqSeekVer failed ver:%"PRId64", %s", pOffset->version + 1, id); return -1; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c120c6055d..f83758b3d8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1586,7 +1586,7 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return -1; } -int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* pTableListInfo, SReadHandle* readHandle) { +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle) { switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { SInserterParam* pInserterParam = taosMemoryCalloc(1, sizeof(SInserterParam)); @@ -1604,23 +1604,26 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* return TSDB_CODE_OUT_OF_MEMORY; } - int32_t tbNum = tableListGetSize(pTableListInfo); + SArray* pInfoList = getTableListInfo(pTask); + STableListInfo* pTableListInfo = taosArrayGetP(pInfoList, 0); + taosArrayDestroy(pInfoList); + pDeleterParam->suid = tableListGetSuid(pTableListInfo); // TODO extract uid list - pDeleterParam->pUidList = taosArrayInit(tbNum, sizeof(uint64_t)); + int32_t numOfTables = tableListGetSize(pTableListInfo); + pDeleterParam->pUidList = taosArrayInit(numOfTables, sizeof(uint64_t)); if (NULL == pDeleterParam->pUidList) { taosMemoryFree(pDeleterParam); return TSDB_CODE_OUT_OF_MEMORY; } - for (int32_t i = 0; i < tbNum; ++i) { + for (int32_t i = 0; i < numOfTables; ++i) { STableKeyInfo* pTable = tableListGetInfo(pTableListInfo, i); taosArrayPush(pDeleterParam->pUidList, &pTable->uid); } *pParam = pDeleterParam; - break; } default: From aa399223e010fb31ae50c672328bbe343ba169d6 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 6 Apr 2023 14:48:20 +0800 Subject: [PATCH 155/176] fix: benchmark from to main (#20784) * fix: taosbenchmark child from/to for main * fix: taosbenchmark create tables from to for main --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index a9e6b57000..8be698e9c9 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 6ae60ac + GIT_TAG 01195d6 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 665ba407cf8b77c84886eebc0394c9b53996af6c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Apr 2023 15:08:35 +0800 Subject: [PATCH 156/176] fix: fix syntax error. --- source/libs/executor/inc/executorimpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index db4c3f0b8d..6e39e0aab5 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -832,7 +832,7 @@ SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo); int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t vgId, char* sql, EOPTR_EXEC_MODEL model); -int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* pTableListInfo, SReadHandle* readHandle); +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, From f8bcdceceab173f9a4b88afd754f686dd0cfe37b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 6 Apr 2023 16:24:04 +0800 Subject: [PATCH 157/176] fix:remove smlChildTableName from tag --- source/client/src/clientSml.c | 1 + utils/test/c/sml_test.c | 42 ++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 40a685faf5..4c28ee76c3 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -174,6 +174,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { memset(childTableName, 0, TSDB_TABLE_NAME_LEN); strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + taosArrayRemove(tags, i); break; } } diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index b0cc6f749c..755ab55625 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -773,26 +773,26 @@ int sml_dup_time_Test() { taos_free_result(pRes); const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -942,6 +942,8 @@ int sml_ts2164_Test() { "ts3038,location=l2a,groupid=ca current=L\"11.8\"", "ts3038,location=l2a,groupid=ca voltage=L\"221\"", "ts3038,location=l2a,groupid=ca phase=L\"221\"", +// "qgyltizmkq,id=sub_table_0123456,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64", +// "qgyltizmkq,id=sub_table_0123456,t=3,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64" // "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; From 0a1476763ab66e3882b6a8f61efdb0e671936217 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 4 Apr 2023 20:01:03 +0800 Subject: [PATCH 158/176] enh: refactor some func names of syncLogReplMgr --- source/libs/sync/inc/syncPipeline.h | 40 ++++---- source/libs/sync/src/syncAppendEntriesReply.c | 2 +- source/libs/sync/src/syncMain.c | 8 +- source/libs/sync/src/syncPipeline.c | 92 +++++++++---------- source/libs/sync/src/syncReplication.c | 4 +- source/libs/sync/src/syncUtil.c | 4 +- 6 files changed, 75 insertions(+), 75 deletions(-) diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index a1de2ee71a..68db811b12 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -59,36 +59,36 @@ typedef struct SSyncLogBuffer { } SSyncLogBuffer; // SSyncLogRepMgr -SSyncLogReplMgr* syncLogReplMgrCreate(); -void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr); -void syncLogReplMgrReset(SSyncLogReplMgr* pMgr); +SSyncLogReplMgr* syncLogReplCreate(); +void syncLogReplDestroy(SSyncLogReplMgr* pMgr); +void syncLogReplReset(SSyncLogReplMgr* pMgr); -int32_t syncNodeLogReplMgrInit(SSyncNode* pNode); -void syncNodeLogReplMgrDestroy(SSyncNode* pNode); +int32_t syncNodeLogReplInit(SSyncNode* pNode); +void syncNodeLogReplDestroy(SSyncNode* pNode); // access -static FORCE_INLINE int64_t syncLogGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) { +static FORCE_INLINE int64_t syncLogReplGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) { return ((int64_t)1 << pMgr->retryBackoff) * SYNC_LOG_REPL_RETRY_WAIT_MS; } -static FORCE_INLINE int32_t syncLogGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { +static FORCE_INLINE int32_t syncLogReplGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { return TMIN(pMgr->retryBackoff + 1, SYNC_MAX_RETRY_BACKOFF); } -SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier); -int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +int32_t syncLogReplReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, + SRaftId* pDestId, bool* pBarrier); +int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); -int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); +int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); // SSyncLogBuffer SSyncLogBuffer* syncLogBufferCreate(); @@ -111,8 +111,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf); int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex); -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, - int32_t applyCode); +int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode); #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index a60f43cd5e..7c343c0e5d 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -85,7 +85,7 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { sError("vgId:%d, failed to get log repl mgr for src addr: 0x%016" PRIx64, ths->vgId, pMsg->srcId.addr); return -1; } - (void)syncLogReplMgrProcessReply(pMgr, ths, pMsg); + (void)syncLogReplProcessReply(pMgr, ths, pMsg); } return 0; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index a617a5d293..c25ea24249 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -949,7 +949,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->changing = false; // replication mgr - if (syncNodeLogReplMgrInit(pSyncNode) < 0) { + if (syncNodeLogReplInit(pSyncNode) < 0) { sError("vgId:%d, failed to init repl mgr since %s.", pSyncNode->vgId, terrstr()); goto _error; } @@ -1122,7 +1122,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncNodeStopPingTimer(pSyncNode); syncNodeStopElectTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode); - syncNodeLogReplMgrDestroy(pSyncNode); + syncNodeLogReplDestroy(pSyncNode); syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -2164,7 +2164,7 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { if (syncLogBufferAppend(ths->pLogBuf, ths, pEntry) < 0) { sError("vgId:%d, failed to enqueue sync log buffer, index:%" PRId64, ths->vgId, pEntry->index); ASSERT(terrno != 0); - (void)syncLogFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno); + (void)syncFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno); syncEntryDestroy(pEntry); return -1; } @@ -2374,7 +2374,7 @@ int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs); - return syncLogReplMgrProcessHeartbeatReply(pMgr, ths, pMsg); + return syncLogReplProcessHeartbeatReply(pMgr, ths, pMsg); } int32_t syncNodeOnHeartbeatReplyOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) { diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 6600b505c1..04e52b3f49 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -86,7 +86,7 @@ _err: return -1; } -SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SSyncRaftEntry* pEntry = NULL; SyncIndex prevIndex = index - 1; @@ -316,7 +316,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt " %" PRId64 ", %" PRId64 ")", pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); - SyncTerm term = syncLogReplMgrGetPrevLogTerm(NULL, pNode, index + 1); + SyncTerm term = syncLogReplGetPrevLogTerm(NULL, pNode, index + 1); ASSERT(pEntry->term >= 0); if (term == pEntry->term) { ret = 0; @@ -351,7 +351,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); - SyncTerm existPrevTerm = syncLogReplMgrGetPrevLogTerm(NULL, pNode, index); + SyncTerm existPrevTerm = syncLogReplGetPrevLogTerm(NULL, pNode, index); ASSERT(pEntry->term == pExist->term && (pEntry->index > pBuf->matchIndex || prevTerm == existPrevTerm)); ret = 0; goto _out; @@ -482,8 +482,8 @@ _out: return matchIndex; } -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, - int32_t applyCode) { +int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode) { if (pNode->replicaNum == 1 && pNode->restoreFinish && pNode->vgId != 1) { return 0; } @@ -564,7 +564,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm pEntry->term, TMSG_INFO(pEntry->originalRpcType)); } - if (syncLogFsmExecute(pNode, pFsm, role, currentTerm, pEntry, 0) != 0) { + if (syncFsmExecute(pNode, pFsm, role, currentTerm, pEntry, 0) != 0) { sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 ", role:%d, current term:%" PRId64, vgId, pEntry->index, pEntry->term, role, currentTerm); @@ -611,7 +611,7 @@ _out: return ret; } -void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { +void syncLogReplReset(SSyncLogReplMgr* pMgr) { if (pMgr == NULL) return; ASSERT(pMgr->startIndex >= 0); @@ -625,14 +625,14 @@ void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { pMgr->retryBackoff = 0; } -int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->endIndex <= pMgr->startIndex) { return 0; } SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; if (pMgr->retryBackoff == SYNC_MAX_RETRY_BACKOFF) { - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); sWarn("vgId:%d, reset sync log repl mgr since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId, pDestId->addr); return -1; @@ -640,7 +640,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { int32_t ret = -1; bool retried = false; - int64_t retryWaitMs = syncLogGetRetryBackoffTimeMs(pMgr); + int64_t retryWaitMs = syncLogReplGetRetryBackoffTimeMs(pMgr); int64_t nowMs = taosGetMonoTimestampMs(); int count = 0; int64_t firstIndex = -1; @@ -657,7 +657,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->states[pos].acked) { if (pMgr->matchIndex < index && pMgr->states[pos].timeMs + (syncGetRetryMaxWaitMs() << 3) < nowMs) { - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); sWarn("vgId:%d, reset sync log repl mgr since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId, index, pDestId->addr); goto _out; @@ -666,7 +666,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { } bool barrier = false; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate sync log entry since %s. index:%" PRId64 ", dest:%" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); goto _out; @@ -687,7 +687,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ret = 0; _out: if (retried) { - pMgr->retryBackoff = syncLogGetNextRetryBackoff(pMgr); + pMgr->retryBackoff = syncLogReplGetNextRetryBackoff(pMgr); SSyncLogBuffer* pBuf = pNode->pLogBuf; sInfo("vgId:%d, resend %d sync log entries. dest:%" PRIx64 ", indexes:%" PRId64 " ..., terms: ... %" PRId64 ", retryWaitMs:%" PRId64 ", mgr: [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 @@ -698,7 +698,7 @@ _out: return ret; } -int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SRaftId destId = pMsg->srcId; ASSERT(pMgr->restored == false); @@ -716,7 +716,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p } } else { if (pMsg->lastSendIndex < pMgr->startIndex || pMsg->lastSendIndex >= pMgr->endIndex) { - syncLogReplMgrRetryOnNeed(pMgr, pNode); + syncLogReplRetryOnNeed(pMgr, pNode); return 0; } @@ -750,7 +750,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p SyncIndex index = TMIN(pMsg->matchIndex, pNode->pLogBuf->matchIndex); if (pMsg->matchIndex < pNode->pLogBuf->matchIndex) { - term = syncLogReplMgrGetPrevLogTerm(pMgr, pNode, index + 1); + term = syncLogReplGetPrevLogTerm(pMgr, pNode, index + 1); if ((index + 1 < firstVer) || (term < 0) || (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) { ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); @@ -773,53 +773,53 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p } // attempt to replicate the raft log at index - (void)syncLogReplMgrReset(pMgr); - return syncLogReplMgrReplicateProbe(pMgr, pNode, index); + (void)syncLogReplReset(pMgr); + return syncLogReplReplicateProbe(pMgr, pNode, index); } -int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { +int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != 0 && pMsg->startTime != pMgr->peerStartTime) { sInfo("vgId:%d, reset sync log repl mgr in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "", pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != pMgr->peerStartTime) { sInfo("vgId:%d, reset sync log repl mgr in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64, pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; } if (pMgr->restored) { - (void)syncLogReplMgrProcessReplyAsNormal(pMgr, pNode, pMsg); + (void)syncLogReplProcessReplyAsNormal(pMgr, pNode, pMsg); } else { - (void)syncLogReplMgrProcessReplyAsRecovery(pMgr, pNode, pMsg); + (void)syncLogReplProcessReplyAsRecovery(pMgr, pNode, pMsg); } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->restored) { - (void)syncLogReplMgrReplicateAttempt(pMgr, pNode); + (void)syncLogReplReplicateAttempt(pMgr, pNode); } else { - (void)syncLogReplMgrReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); + (void)syncLogReplReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); } return 0; } -int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +int32_t syncLogReplReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { ASSERT(!pMgr->restored); ASSERT(pMgr->startIndex >= 0); int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs(); @@ -829,12 +829,12 @@ int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy nowMs < pMgr->states[pMgr->startIndex % pMgr->size].timeMs + retryMaxWaitMs) { return 0; } - (void)syncLogReplMgrReset(pMgr); + (void)syncLogReplReset(pMgr); SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -857,7 +857,7 @@ int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy return 0; } -int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ASSERT(pMgr->restored); SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; @@ -879,7 +879,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -902,7 +902,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) } } - syncLogReplMgrRetryOnNeed(pMgr, pNode); + syncLogReplRetryOnNeed(pMgr, pNode); SSyncLogBuffer* pBuf = pNode->pLogBuf; sTrace("vgId:%d, replicated %d msgs to peer:%" PRIx64 ". indexes:%" PRId64 "..., terms: ...%" PRId64 @@ -913,7 +913,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) return 0; } -int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { ASSERT(pMgr->restored == true); if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { @@ -932,10 +932,10 @@ int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNo pMgr->startIndex = pMgr->matchIndex; } - return syncLogReplMgrReplicateAttempt(pMgr, pNode); + return syncLogReplReplicateAttempt(pMgr, pNode); } -SSyncLogReplMgr* syncLogReplMgrCreate() { +SSyncLogReplMgr* syncLogReplCreate() { SSyncLogReplMgr* pMgr = taosMemoryCalloc(1, sizeof(SSyncLogReplMgr)); if (pMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -949,7 +949,7 @@ SSyncLogReplMgr* syncLogReplMgrCreate() { return pMgr; } -void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr) { +void syncLogReplDestroy(SSyncLogReplMgr* pMgr) { if (pMgr == NULL) { return; } @@ -957,10 +957,10 @@ void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr) { return; } -int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { +int32_t syncNodeLogReplInit(SSyncNode* pNode) { for (int i = 0; i < TSDB_MAX_REPLICA; i++) { ASSERT(pNode->logReplMgrs[i] == NULL); - pNode->logReplMgrs[i] = syncLogReplMgrCreate(); + pNode->logReplMgrs[i] = syncLogReplCreate(); if (pNode->logReplMgrs[i] == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -970,9 +970,9 @@ int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { return 0; } -void syncNodeLogReplMgrDestroy(SSyncNode* pNode) { +void syncNodeLogReplDestroy(SSyncNode* pNode) { for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - syncLogReplMgrDestroy(pNode->logReplMgrs[i]); + syncLogReplDestroy(pNode->logReplMgrs[i]); pNode->logReplMgrs[i] = NULL; } } @@ -1103,7 +1103,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { // reset repl mgr for (int i = 0; i < pNode->replicaNum; i++) { SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); } syncLogBufferValidate(pBuf); taosThreadMutexUnlock(&pBuf->mutex); @@ -1127,8 +1127,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, return pEntry; } -int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier) { +int32_t syncLogReplReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, + SRaftId* pDestId, bool* pBarrier) { SSyncRaftEntry* pEntry = NULL; SRpcMsg msgOut = {0}; bool inBuf = false; @@ -1143,14 +1143,14 @@ int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy if (pMgr) { sInfo("vgId:%d, reset sync log repl mgr of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId, pDestId->addr, terrstr(), index); - (void)syncLogReplMgrReset(pMgr); + (void)syncLogReplReset(pMgr); } } goto _err; } *pBarrier = syncLogIsReplicationBarrier(pEntry); - prevLogTerm = syncLogReplMgrGetPrevLogTerm(pMgr, pNode, index); + prevLogTerm = syncLogReplGetPrevLogTerm(pMgr, pNode, index); if (prevLogTerm < 0) { sError("vgId:%d, failed to get prev log term since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); goto _err; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 1d94b288d3..43d2bc839b 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -52,7 +52,7 @@ int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); taosThreadMutexUnlock(&pBuf->mutex); return 0; } @@ -74,7 +74,7 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { continue; } SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - (void)syncLogReplMgrReplicateOnce(pMgr, pNode); + (void)syncLogReplReplicateOnce(pMgr, pNode); } return 0; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index a519c76cda..056a597777 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -125,7 +125,7 @@ static void syncLogBufferStates2Str(SSyncNode* pSyncNode, char* buf, int32_t buf pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); } -static void syncLogReplMgrStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { +static void syncLogReplStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int len = 0; len += snprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; i++) { @@ -178,7 +178,7 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr)); char replMgrStatesStr[1024] = ""; - syncLogReplMgrStates2Str(pNode, replMgrStatesStr, sizeof(replMgrStatesStr)); + syncLogReplStates2Str(pNode, replMgrStatesStr, sizeof(replMgrStatesStr)); char bufferStatesStr[256] = ""; syncLogBufferStates2Str(pNode, bufferStatesStr, sizeof(bufferStatesStr)); From fa154172b40786106b2cb1aad61c0d9c4149ac1d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 6 Apr 2023 18:15:03 +0800 Subject: [PATCH 159/176] fix:time_wait in doAskEp --- source/client/src/clientTmq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index e295ec93af..0e0cfaa94d 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1529,8 +1529,8 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { } else { tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, head->epoch, epoch); - pParam->pUserFn(tmq, code, pMsg, pParam->pParam); } + pParam->pUserFn(tmq, code, pMsg, pParam->pParam); taosReleaseRef(tmqMgmt.rsetId, pParam->refId); From 69bf3625dcb7516cd03485effce64f07195f6664 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 6 Apr 2023 18:32:28 +0800 Subject: [PATCH 160/176] fix: install script (#20794) * fix: install.sh * fix: install.sh --- packaging/tools/install.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index a3f8b53d33..3ff59498ba 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -79,6 +79,13 @@ if command -v sudo >/dev/null; then csudo="sudo " fi +if command -v install >/dev/null; then + echo "" > /dev/null +else + echo -e "${RED} Not found install utility, exit! ${NC}" + exit 1 +fi + update_flag=0 prompt_force=0 From 7a58f448a5011200a9b9e43305a522882445695f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 6 Apr 2023 19:16:31 +0800 Subject: [PATCH 161/176] fix:time_wait in doAskEp --- source/client/src/clientTmq.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 0e0cfaa94d..befcb00ac7 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1426,6 +1426,9 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; tscDebug("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur); + if (epoch <= tmq->epoch) { + return false; + } SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); if (newTopics == NULL) { From 360113115f2311601e4d9763e7bb271cba1b6dca Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Apr 2023 09:49:48 +0800 Subject: [PATCH 162/176] fix(tsdb/row merger): remove unused code --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index c323ae1532..e4592d2758 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -727,23 +727,6 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); if (key.version > pMerger->version) { -#if 0 - if (!COL_VAL_IS_NONE(pColVal)) { - if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { - SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol); - code = tRealloc(&tColVal->value.pData, pColVal->value.nData); - if (code) return code; - - tColVal->value.nData = pColVal->value.nData; - if (pColVal->value.nData) { - memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); - } - tColVal->flag = 0; - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); - } - } -#endif if (!COL_VAL_IS_NONE(pColVal)) { if (IS_VAR_DATA_TYPE(pColVal->type)) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); @@ -758,7 +741,6 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pTColVal->flag = 0; } else { tFree(pTColVal->value.pData); - pTColVal->value.pData = NULL; taosArraySet(pMerger->pArray, iCol, pColVal); } } else { From 50ced8a057951152c85e2f417a9291473b24e20e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 7 Apr 2023 13:22:42 +0800 Subject: [PATCH 163/176] fix: taosbenchmark from/to neg value for main (#20805) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 8be698e9c9..b427177e5b 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 01195d6 + GIT_TAG 63635fc SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 55f7e41f3b1a49ed185de436bc57ff9bb5784e7d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Apr 2023 13:31:33 +0800 Subject: [PATCH 164/176] tsdb/row merger: remove tsdbRowMerge --- source/dnode/vnode/src/inc/tsdb.h | 4 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 146 ++++++++++++------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 7 +- 3 files changed, 79 insertions(+), 78 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0c4ada2cb1..0dd767990e 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -128,7 +128,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); void tsdbRowMergerClear(SRowMerger *pMerger); -int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); +// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); @@ -224,7 +224,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, void *tsdbTbDataIterDestroy(STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter); -void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum); +void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1ac327de7f..338231057c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -959,14 +959,15 @@ static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int } static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, - SBlockLoadSuppInfo* pSup) { + SBlockLoadSuppInfo* pSup) { if (IS_VAR_DATA_TYPE(pColVal->type)) { if (!COL_VAL_IS_VALUE(pColVal)) { colDataSetNULL(pColInfoData, rowIndex); } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); if (pColVal->value.nData > pColInfoData->info.bytes) { - tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData, pColInfoData->info.bytes); + tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData, + pColInfoData->info.bytes); return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; } if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 @@ -1794,7 +1795,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* } static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key, - SFileBlockDumpInfo* pDumpInfo, bool *copied) { + SFileBlockDumpInfo* pDumpInfo, bool* copied) { // opt version // 1. it is not a border point // 2. the direct next point is not an duplicated timestamp @@ -1843,7 +1844,8 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, - STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, bool *copied) { + STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, + bool* copied) { int32_t code = TSDB_CODE_SUCCESS; *copied = false; @@ -1856,7 +1858,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas if (code) { return code; } - + *copied = true; return code; } @@ -1865,7 +1867,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas if (code) { return code; } - + *copied = true; return code; } @@ -1977,7 +1979,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); @@ -2025,7 +2027,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); @@ -2038,7 +2040,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { if (init) { - tsdbRowMerge(&merge, &fRow); + tsdbRowMergerAdd(&merge, &fRow, NULL); } else { init = true; int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); @@ -2068,11 +2070,11 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, bool mergeBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - bool copied = false; - int32_t code = TSDB_CODE_SUCCESS; - SRow* pTSRow = NULL; - SRowMerger merge = {0}; - TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + bool copied = false; + int32_t code = TSDB_CODE_SUCCESS; + SRow* pTSRow = NULL; + SRowMerger merge = {0}; + TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); // only last block exists @@ -2081,7 +2083,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, if (code) { return code; } - + if (copied) { pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; @@ -2092,7 +2094,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr); code = tsdbRowMergerGetRow(&merge, &pTSRow); @@ -2108,7 +2110,6 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, if (code != TSDB_CODE_SUCCESS) { return code; } - } } else { // not merge block data int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); @@ -2171,7 +2172,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange, pReader->idStr); @@ -2273,7 +2274,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); @@ -2287,7 +2288,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, NULL); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); @@ -2314,7 +2315,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - tsdbRowMerge(&merge, pRow); + tsdbRowMergerAdd(&merge, pRow, NULL); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); code = tsdbRowMergerInit(&merge, pRow, pSchema); @@ -2346,7 +2347,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, NULL); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); @@ -2365,7 +2366,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); @@ -2387,7 +2388,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (merge.pTSchema == NULL) { return code; } - tsdbRowMerge(&merge, &fRow); + tsdbRowMergerAdd(&merge, &fRow, NULL); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -2557,12 +2558,12 @@ bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - bool copied = false; - int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied); + bool copied = false; + int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied); if (code) { return code; } - + if (copied) { pBlockScanInfo->lastKey = key; return TSDB_CODE_SUCCESS; @@ -2758,7 +2759,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { if (code) { goto _end; } - + // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); @@ -2776,8 +2777,8 @@ _end: updateComposedBlockInfo(pReader, el, pBlockScanInfo); if (pResBlock->info.rows > 0) { - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%" PRId64 ", elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -3018,7 +3019,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { if (code) { return code; } - + if (pResBlock->info.rows >= pReader->capacity) { break; } @@ -3028,8 +3029,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { updateComposedBlockInfo(pReader, el, pScanInfo); if (pResBlock->info.rows > 0) { - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%" PRId64 ", elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -3102,7 +3103,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (code) { return code; } - + if (pResBlock->info.rows >= pReader->capacity) { break; } @@ -3112,8 +3113,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { updateComposedBlockInfo(pReader, el, pScanInfo); if (pResBlock->info.rows > 0) { - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%" PRId64 ", elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -3139,7 +3140,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } - static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { int64_t st = taosGetTimestampUs(); LRUHandle* handle = NULL; @@ -3157,8 +3157,8 @@ static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReade return TSDB_CODE_SUCCESS; } - SBlockIdx* pBlockIdx = NULL; - int32_t i = 0; + SBlockIdx* pBlockIdx = NULL; + int32_t i = 0; for (int32_t i = 0; i < num; ++i) { pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); if (pBlockIdx->suid != pReader->suid) { @@ -3170,7 +3170,7 @@ static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReade continue; } - STableBlockScanInfo *pScanInfo = *p; + STableBlockScanInfo* pScanInfo = *p; tMapDataReset(&pScanInfo->mapData); tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); @@ -3186,15 +3186,14 @@ _end: return code; } - static int32_t doSumSttBlockRows(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; - SSttBlockLoadInfo* pBlockLoadInfo = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockLoadInfo* pBlockLoadInfo = NULL; for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file pBlockLoadInfo = &pLastBlockReader->pInfo[i]; - + code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); if (code) { return code; @@ -3202,9 +3201,9 @@ static int32_t doSumSttBlockRows(STsdbReader* pReader) { size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); if (size >= 1) { - SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); - SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); - + SSttBlk* pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); + SSttBlk* pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); + // all identical if (pStart->suid == pEnd->suid) { if (pStart->suid != pReader->suid) { @@ -3213,17 +3212,17 @@ static int32_t doSumSttBlockRows(STsdbReader* pReader) { continue; } for (int32_t i = 0; i < size; ++i) { - SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); pReader->rowsNum += p->nRow; } } else { for (int32_t i = 0; i < size; ++i) { - SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); uint64_t s = p->suid; if (s < pReader->suid) { continue; } - + if (s == pReader->suid) { pReader->rowsNum += p->nRow; } else if (s > pReader->suid) { @@ -3238,7 +3237,7 @@ static int32_t doSumSttBlockRows(STsdbReader* pReader) { } static int32_t readRowsCountFromFiles(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; while (1) { bool hasNext = false; @@ -3259,7 +3258,7 @@ static int32_t readRowsCountFromFiles(STsdbReader* pReader) { code = doSumSttBlockRows(pReader); if (code != TSDB_CODE_SUCCESS) { return code; - } + } } pReader->status.loadFromFile = false; @@ -3268,8 +3267,8 @@ static int32_t readRowsCountFromFiles(STsdbReader* pReader) { } static int32_t readRowsCountFromMem(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - int64_t memNum = 0, imemNum = 0; + int32_t code = TSDB_CODE_SUCCESS; + int64_t memNum = 0, imemNum = 0; if (pReader->pReadSnap->pMem != NULL) { tsdbMemTableCountRows(pReader->pReadSnap->pMem, pReader->status.pTableMap, &memNum); } @@ -3283,7 +3282,6 @@ static int32_t readRowsCountFromMem(STsdbReader* pReader) { return code; } - static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; STableUidList* pUidList = &pStatus->uidList; @@ -3696,7 +3694,7 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe tsdbRowMergerAdd(pMerger, pRow, pTSchema); } else { // column format - tsdbRowMerge(pMerger, pRow); + tsdbRowMergerAdd(pMerger, pRow, NULL); } } @@ -3712,7 +3710,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tsdbRowMerge(pMerger, &fRow); + tsdbRowMergerAdd(pMerger, &fRow, NULL); rowIndex += step; } @@ -3790,7 +3788,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(pMerger, &fRow1); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt, @@ -3863,7 +3861,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return code; } - tsdbRowMerge(&merge, pNextRow); + tsdbRowMergerAdd(&merge, pNextRow, NULL); } code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); @@ -3926,7 +3924,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, NULL); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -4000,7 +3998,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; int64_t uid = pScanInfo->uid; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); @@ -4106,7 +4104,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader) { SSDataBlock* pBlock = pReader->pResBlock; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; do { // SRow* pTSRow = NULL; @@ -4342,7 +4340,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL if (countOnly) { pReader->readMode = READ_MODE_COUNT_ONLY; } - + tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -4644,7 +4642,7 @@ _err: } static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SSDataBlock* pBlock = pReader->pResBlock; if (pReader->status.loadFromFile == false) { @@ -4664,15 +4662,15 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { pBlock->info.rows = pReader->rowsNum; pBlock->info.id.uid = 0; pBlock->info.dataLoad = 0; - + pReader->rowsNum = 0; - + return pBlock->info.rows > 0; } -static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { +static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { int32_t code = TSDB_CODE_SUCCESS; - + // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; blockDataCleanup(pBlock); @@ -4707,11 +4705,11 @@ static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { return code; } -int32_t tsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { +int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { int32_t code = TSDB_CODE_SUCCESS; *hasNext = false; - + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT) { return code; } @@ -4731,7 +4729,7 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { tsdbReleaseReader(pReader); return code; } - + pReader->step = EXTERNAL_ROWS_PREV; if (*hasNext) { pStatus = &pReader->innerReader[0]->status; @@ -4762,7 +4760,7 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { tsdbReleaseReader(pReader); return code; } - + if (*hasNext) { if (pStatus->composedDataBlock) { qTrace("tsdb/read: %p, unlock read mutex", pReader); @@ -4786,7 +4784,7 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool *hasNext) { tsdbReleaseReader(pReader); return code; } - + pReader->step = EXTERNAL_ROWS_NEXT; if (*hasNext) { pStatus = &pReader->innerReader[1]->status; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index e4592d2758..9aeea22b99 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -712,6 +712,9 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) STColumn *pTColumn; int32_t iCol, jCol = 1; + if (NULL == pTSchema) { + pTSchema = pMerger->pTSchema; + } ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { @@ -833,7 +836,7 @@ void tsdbRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } - +/* int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); @@ -898,7 +901,7 @@ int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { _exit: return code; } - +*/ int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } From 1ff8511eb3af8b500a139571bbf6099b14abaf40 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Apr 2023 13:43:07 +0800 Subject: [PATCH 165/176] tsdb/row merger: comment init out --- source/dnode/vnode/src/inc/tsdb.h | 6 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 44 +++++++++++++------------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 10 ++++-- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0dd767990e..2a85b191a4 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -123,11 +123,11 @@ int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) void tsdbRowClose(STSDBRowIter *pIter); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger -int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tsdbRowMergerClear(SRowMerger *pMerger); +// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowMergerClear(SRowMerger *pMerger); // int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 338231057c..55cbf2c847 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1969,7 +1969,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1982,7 +1982,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1999,7 +1999,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2013,7 +2013,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2030,7 +2030,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2043,7 +2043,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2088,7 +2088,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2112,7 +2112,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2164,7 +2164,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2263,7 +2263,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2277,7 +2277,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2296,7 +2296,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tsdbRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2318,7 +2318,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, NULL); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2333,7 +2333,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2351,7 +2351,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2369,7 +2369,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2380,7 +2380,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2573,7 +2573,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3844,7 +3844,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, pReader->pSchema = pTSchema; } - code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + code = tsdbRowMergerInit(&merge, pReader->pSchema, ¤t, pTSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3856,7 +3856,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); } else { // let's merge rows in file block - code = tsdbRowMergerInit(&merge, ¤t, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, ¤t, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3892,7 +3892,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit2(&merge, pSchema, piRow, piSchema); + int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3913,7 +3913,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 9aeea22b99..dd11134bd0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -638,13 +638,17 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { // SRowMerger ====================================================== -int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; STColumn *pTColumn; int32_t iCol, jCol = 0; + if (NULL == pResTSchema) { + pResTSchema = pTSchema; + } + pMerger->pTSchema = pResTSchema; pMerger->version = key.version; @@ -774,7 +778,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pMerger->version = key.version; return code; } - +/* int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); @@ -825,7 +829,7 @@ int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema _exit: return code; } - +*/ void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); From 05ec2b09287bb465d19d10dbc35cf94244d31821 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 7 Apr 2023 14:24:29 +0800 Subject: [PATCH 166/176] fix:[TD-23517] set offset type to wal if only meta & check if tag is the same type in schemaless --- source/dnode/vnode/src/tq/tqScan.c | 2 +- source/libs/executor/src/executor.c | 4 +++- source/libs/parser/src/parInsertSml.c | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index b4e50312fd..a26f59f72b 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -170,7 +170,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta qStreamExtractOffset(task, &tmp->rspOffset); *pMetaRsp = *tmp; - tqDebug("tmqsnap task get data"); + tqDebug("tmqsnap task get meta"); break; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b4431a7c3b..1670eb3c59 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1200,7 +1200,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tableListClear(pTableListInfo); if (mtInfo.uid == 0) { - return 0; // no data + goto end; // no data } initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); @@ -1234,6 +1234,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } } + +end: pTaskInfo->streamInfo.currentOffset = *pOffset; return 0; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 0bcd777d2a..1c921b2d7c 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -125,6 +125,12 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); + if(kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type){ + code = TSDB_CODE_SML_INVALID_DATA; + uError("SML smlBuildCol error col not same %s", pTagSchema->name); + goto end; + } + taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); From f262ac82308394f67080290a9df65a16c0dd08f7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Apr 2023 14:37:56 +0800 Subject: [PATCH 167/176] fix(tsdb/read): use irow's schema instead of NULL --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 55cbf2c847..0cf28c1ed1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3885,13 +3885,12 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p SRow** pTSRow) { SRowMerger merge = {0}; - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3911,8 +3910,6 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } } else { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; @@ -3924,7 +3921,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMergerAdd(&merge, piRow, NULL); + tsdbRowMergerAdd(&merge, piRow, piSchema); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { From 4fb5390c6fecd8a4670ee3bbdb225db4deee8e0d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 7 Apr 2023 17:10:39 +0800 Subject: [PATCH 168/176] chore: release script community main (#20808) * chore: cus name support in script * chore: merge with 3.0 --- include/os/osDir.h | 28 +++++ include/util/cus_name.h | 31 +++++ packaging/cfg/taos.cfg | 15 ++- packaging/cfg/taosd.service | 2 +- packaging/tools/install.sh | 194 +++++++++++++++---------------- packaging/tools/makeclient.sh | 16 ++- packaging/tools/makepkg.sh | 28 +++-- packaging/tools/remove.sh | 41 ++++--- packaging/tools/remove_client.sh | 18 +-- source/client/src/clientEnv.c | 14 ++- source/common/src/tglobal.c | 4 + source/dnode/mgmt/exe/dmMain.c | 17 ++- source/os/src/osSysinfo.c | 12 ++ tools/shell/CMakeLists.txt | 4 - tools/shell/src/shellArguments.c | 31 +++-- 15 files changed, 291 insertions(+), 164 deletions(-) create mode 100644 include/util/cus_name.h diff --git a/include/os/osDir.h b/include/os/osDir.h index 73871602c5..55c7a15764 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -31,21 +31,49 @@ extern "C" { #endif +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #ifdef WINDOWS + #define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\" +#ifdef CUS_NAME +#define TD_CFG_DIR_PATH "C:\\"CUS_NAME"\\cfg\\" +#define TD_DATA_DIR_PATH "C:\\"CUS_NAME"\\data\\" +#define TD_LOG_DIR_PATH "C:\\"CUS_NAME"\\log\\" +#else #define TD_CFG_DIR_PATH "C:\\TDengine\\cfg\\" #define TD_DATA_DIR_PATH "C:\\TDengine\\data\\" #define TD_LOG_DIR_PATH "C:\\TDengine\\log\\" +#endif // CUS_NAME + #elif defined(_TD_DARWIN_64) + +#ifdef CUS_PROMPT +#define TD_TMP_DIR_PATH "/tmp/"CUS_PROMPT"d/" +#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/" +#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/" +#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/" +#else #define TD_TMP_DIR_PATH "/tmp/taosd/" #define TD_CFG_DIR_PATH "/etc/taos/" #define TD_DATA_DIR_PATH "/var/lib/taos/" #define TD_LOG_DIR_PATH "/var/log/taos/" +#endif // CUS_PROMPT + #else + #define TD_TMP_DIR_PATH "/tmp/" +#ifdef CUS_PROMPT +#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/" +#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/" +#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/" +#else #define TD_CFG_DIR_PATH "/etc/taos/" #define TD_DATA_DIR_PATH "/var/lib/taos/" #define TD_LOG_DIR_PATH "/var/log/taos/" +#endif // CUS_PROMPT #endif typedef struct TdDir *TdDirPtr; diff --git a/include/util/cus_name.h b/include/util/cus_name.h new file mode 100644 index 0000000000..16f677f855 --- /dev/null +++ b/include/util/cus_name.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _CUS_NAME_H_ +#define _CUS_NAME_H_ + +#ifndef CUS_NAME +#define CUS_NAME "TDengine" +#endif + +#ifndef CUS_PROMPT +#define CUS_PROMPT "taos" +#endif + +#ifndef CUS_EMAIL +#define CUS_EMAIL "" +#endif + +#endif // _CUS_NAME_H_ diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index a98dc5a236..2159899aa2 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -1,7 +1,6 @@ ######################################################## # # # Configuration # -# Any questions, please email support@taosdata.com # # # ######################################################## @@ -13,7 +12,7 @@ ############### 1. Cluster End point ############################ -# The end point of the first dnode in the cluster to be connected to when this dnode or a CLI `taos` is started +# The end point of the first dnode in the cluster to be connected to when this dnode or the CLI utility is started # firstEp hostname:6030 # The end point of the second dnode to be connected to if the firstEp is not available @@ -25,7 +24,7 @@ # The FQDN of the host on which this dnode will be started. It can be IP address # fqdn hostname -# The port for external access after this dnode is started +# The port for external access after this dnode is started # serverPort 6030 # The maximum number of connections a dnode can accept @@ -96,7 +95,7 @@ # if free disk space is less than this value, this dnode will fail to start # minimalDataDirGB 2.0 -# enable/disable system monitor +# enable/disable system monitor # monitor 1 # The following parameter is used to limit the maximum number of lines in log files. @@ -114,8 +113,8 @@ # The following parameters are used for debug purpose only by this dnode. # debugFlag is a 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR -# Available debug levels are: -# 131: output warning and error +# Available debug levels are: +# 131: output warning and error # 135: output debug, warning and error # 143: output trace, debug, warning and error to log # 199: output debug, warning and error to both screen and file @@ -130,7 +129,7 @@ # debug flag for util # uDebugFlag 131 -# debug flag for rpc +# debug flag for rpc # rpcDebugFlag 131 # debug flag for jni @@ -139,7 +138,7 @@ # debug flag for query # qDebugFlag 131 -# debug flag for taosc driver +# debug flag for client driver # cDebugFlag 131 # debug flag for dnode messages diff --git a/packaging/cfg/taosd.service b/packaging/cfg/taosd.service index fff4b74e62..52c4b1d1e2 100644 --- a/packaging/cfg/taosd.service +++ b/packaging/cfg/taosd.service @@ -1,5 +1,5 @@ [Unit] -Description=TDengine server service +Description=server service After=network-online.target Wants=network-online.target diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 3ff59498ba..858a6ac668 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -4,7 +4,7 @@ # is required to use systemd to manage services at boot set -e -#set -x +# set -x verMode=edge pagMode=full @@ -34,21 +34,25 @@ benchmarkName="taosBenchmark" dumpName="taosdump" demoName="taosdemo" xname="taosx" -explorerName="${clientName}-explorer" clientName2="taos" -serverName2="taosd" +serverName2="${clientName2}d" +configFile2="${clientName2}.cfg" productName2="TDengine" emailName2="taosdata.com" +xname2="${clientName2}x" +adapterName2="${clientName2}adapter" +explorerName="${clientName2}-explorer" benchmarkName2="${clientName2}Benchmark" +demoName2="${clientName2}demo" dumpName2="${clientName2}dump" uninstallScript2="rm${clientName2}" historyFile="${clientName2}_history" logDir="/var/log/${clientName2}" configDir="/etc/${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" data_dir=${dataDir} log_dir=${logDir} @@ -79,13 +83,6 @@ if command -v sudo >/dev/null; then csudo="sudo " fi -if command -v install >/dev/null; then - echo "" > /dev/null -else - echo -e "${RED} Not found install utility, exit! ${NC}" - exit 1 -fi - update_flag=0 prompt_force=0 @@ -213,15 +210,15 @@ function install_main_path() { function install_bin() { # Remove links - ${csudo}rm -f ${bin_link_dir}/${clientName} || : - ${csudo}rm -f ${bin_link_dir}/${serverName} || : + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${serverName2} || : ${csudo}rm -f ${bin_link_dir}/${udfdName} || : ${csudo}rm -f ${bin_link_dir}/${adapterName} || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : - ${csudo}rm -f ${bin_link_dir}/${demoName} || : - ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || : - ${csudo}rm -f ${bin_link_dir}/${dumpName} || : - ${csudo}rm -f ${bin_link_dir}/${xname} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${xname2} || : ${csudo}rm -f ${bin_link_dir}/${explorerName} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -229,24 +226,23 @@ function install_bin() { ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : + [ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || : + [ -x ${install_main_dir}/bin/${serverName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName2} ${bin_link_dir}/${serverName2} || : [ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || : - [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : - [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : - [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || : + [ -x ${install_main_dir}/bin/${adapterName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName2} ${bin_link_dir}/${adapterName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${demoName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : + [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || : + [ -x ${install_main_dir}/bin/${xname2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname2} ${bin_link_dir}/${xname2} || : [ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : - [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + if [ "$clientName2" == "${clientName}" ]; then + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + fi + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName2} || : - [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName2} || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || : fi } @@ -406,7 +402,7 @@ function set_hostname() { ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || : fi - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile2} serverFqdn=$newHostname if [[ -e /etc/hosts ]]; then @@ -440,7 +436,7 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} serverFqdn=$localFqdn echo return @@ -462,7 +458,7 @@ function set_ipAsFqdn() { read -p "Please choose an IP from local IP list:" localFqdn else # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} serverFqdn=$localFqdn break fi @@ -526,15 +522,15 @@ function install_adapter_config() { function install_config() { - if [ ! -f "${cfg_install_dir}/${configFile}" ]; then + if [ ! -f "${cfg_install_dir}/${configFile2}" ]; then ${csudo}mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} + [ -f ${script_dir}/cfg/${configFile2} ] && ${csudo}cp ${script_dir}/cfg/${configFile2} ${cfg_install_dir} ${csudo}chmod 644 ${cfg_install_dir}/* else - ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new + ${csudo}cp -f ${script_dir}/cfg/${configFile2} ${cfg_install_dir}/${configFile2}.new fi - ${csudo}ln -sf ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg + ${csudo}ln -sf ${cfg_install_dir}/${configFile2} ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client @@ -555,7 +551,7 @@ function install_config() { read firstEp while true; do if [ ! -z "$firstEp" ]; then - ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile2} break else break @@ -607,8 +603,8 @@ function install_web() { function clean_service_on_sysvinit() { - if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then - ${csudo}service ${serverName} stop || : + if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then + ${csudo}service ${serverName2} stop || : fi if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then @@ -616,30 +612,30 @@ function clean_service_on_sysvinit() { fi if ((${initd_mod} == 1)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}chkconfig --del ${serverName} || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}chkconfig --del ${serverName2} || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}chkconfig --del tarbitratord || : fi elif ((${initd_mod} == 2)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}insserv -r ${serverName} || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}insserv -r ${serverName2} || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}insserv -r tarbitratord || : fi elif ((${initd_mod} == 3)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}update-rc.d -f ${serverName} remove || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}update-rc.d -f ${serverName2} remove || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}update-rc.d -f tarbitratord remove || : fi fi - ${csudo}rm -f ${service_config_dir}/${serverName} || : + ${csudo}rm -f ${service_config_dir}/${serverName2} || : ${csudo}rm -f ${service_config_dir}/tarbitratord || : if $(which init &>/dev/null); then @@ -660,24 +656,24 @@ function install_service_on_sysvinit() { fi if ((${initd_mod} == 1)); then - ${csudo}chkconfig --add ${serverName} || : - ${csudo}chkconfig --level 2345 ${serverName} on || : + ${csudo}chkconfig --add ${serverName2} || : + ${csudo}chkconfig --level 2345 ${serverName2} on || : elif ((${initd_mod} == 2)); then - ${csudo}insserv ${serverName} || : - ${csudo}insserv -d ${serverName} || : + ${csudo}insserv ${serverName2} || : + ${csudo}insserv -d ${serverName2} || : elif ((${initd_mod} == 3)); then - ${csudo}update-rc.d ${serverName} defaults || : + ${csudo}update-rc.d ${serverName2} defaults || : fi } function clean_service_on_systemd() { - taosd_service_config="${service_config_dir}/${serverName}.service" - if systemctl is-active --quiet ${serverName}; then + service_config="${service_config_dir}/${serverName2}.service" + if systemctl is-active --quiet ${serverName2}; then echo "${productName} is running, stopping it..." - ${csudo}systemctl stop ${serverName} &>/dev/null || echo &>/dev/null + ${csudo}systemctl stop ${serverName2} &>/dev/null || echo &>/dev/null fi - ${csudo}systemctl disable ${serverName} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${taosd_service_config} + ${csudo}systemctl disable ${serverName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${service_config} tarbitratord_service_config="${service_config_dir}/tarbitratord.service" if systemctl is-active --quiet tarbitratord; then @@ -694,19 +690,19 @@ function clean_service_on_systemd() { function install_service_on_systemd() { clean_service_on_systemd - [ -f ${script_dir}/cfg/${serverName}.service ] && - ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + [ -f ${script_dir}/cfg/${serverName2}.service ] && + ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ ${service_config_dir}/ || : # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - # [ -f ${script_dir}/cfg/${serverName}.service ] && - # ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + # [ -f ${script_dir}/cfg/${serverName2}.service ] && + # ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ # ${service_config_dir}/${serverName2}.service || : # fi ${csudo}systemctl daemon-reload - ${csudo}systemctl enable ${serverName} + ${csudo}systemctl enable ${serverName2} ${csudo}systemctl daemon-reload } @@ -726,7 +722,7 @@ function install_service() { elif ((${service_mod} == 1)); then install_service_on_sysvinit else - kill_process ${serverName} + kill_process ${serverName2} fi } @@ -763,10 +759,10 @@ function is_version_compatible() { if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt) else - min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5) + min_compatible_version=$(${script_dir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 5) fi - exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3) + exist_version=$(${installDir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 3) vercomp $exist_version "3.0.0.0" case $? in 2) @@ -836,13 +832,13 @@ function updateProduct() { echo -e "${GREEN}Start to update ${productName2}...${NC}" # Stop the service if running - if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then if ((${service_mod} == 0)); then - ${csudo}systemctl stop ${serverName} || : + ${csudo}systemctl stop ${serverName2} || : elif ((${service_mod} == 1)); then - ${csudo}service ${serverName} stop || : + ${csudo}service ${serverName2} stop || : else - kill_process ${serverName} + kill_process ${serverName2} fi sleep 1 fi @@ -869,21 +865,21 @@ function updateProduct() { openresty_work=false echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}" - [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName}${NC}" - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName2}${NC}" + [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" fi if [ ${openresty_work} = 'true' ]; then @@ -894,7 +890,7 @@ function updateProduct() { if ((${prompt_force} == 1)); then echo "" - echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}" + echo -e "${RED}Please run '${serverName2} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}" fi echo echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}" @@ -906,7 +902,7 @@ function updateProduct() { echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}" fi - rm -rf $(tar -tf ${tarName} | grep -v "^\./$") + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } function installProduct() { @@ -951,21 +947,21 @@ function installProduct() { # Ask if to start the service echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile}" - [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName}${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName} start${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName}${NC}" - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName2}${NC}" + [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" fi if [ ! -z "$firstEp" ]; then @@ -998,7 +994,7 @@ function installProduct() { fi touch ~/.${historyFile} - rm -rf $(tar -tf ${tarName} | grep -v "^\./$") + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } ## ==============================Main program starts from here============================ @@ -1009,7 +1005,7 @@ if [ "$verType" == "server" ]; then echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of ${productName2} 2.x, please clear it before installing!\033[0m" else # Install server and client - if [ -x ${bin_dir}/${serverName} ]; then + if [ -x ${bin_dir}/${serverName2} ]; then update_flag=1 updateProduct else @@ -1019,7 +1015,7 @@ if [ "$verType" == "server" ]; then elif [ "$verType" == "client" ]; then interactiveFqdn=no # Only install client - if [ -x ${bin_dir}/${clientName} ]; then + if [ -x ${bin_dir}/${clientName2} ]; then update_flag=1 updateProduct client else diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index f46de0f94b..9413f55d51 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -23,9 +23,12 @@ clientName2="${12}" productName="TDengine" clientName="taos" +benchmarkName="taosBenchmark" configFile="taos.cfg" tarName="package.tar.gz" +benchmarkName2="${clientName2}Benchmark" + if [ "$osType" != "Darwin" ]; then script_dir="$(dirname $(readlink -f $0))" top_dir="$(readlink -f ${script_dir}/../..)" @@ -53,11 +56,12 @@ fi # Directories and files. -if [ "$verMode" == "cluster" ]; then - sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh - sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh - sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh -fi +#if [ "$verMode" == "cluster" ]; then +# sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh +# sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh +# sed -i "s/configFile2=\"taos\"/configFile2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh +# sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh +#fi if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then @@ -66,6 +70,7 @@ if [ "$osType" != "Darwin" ]; then ${script_dir}/remove_client.sh" else bin_files="${build_dir}/bin/${clientName} \ + ${build_dir}/bin/${benchmarkName} \ ${script_dir}/remove_client.sh \ ${script_dir}/set_core.sh \ ${script_dir}/get_client.sh" @@ -153,6 +158,7 @@ if [ "$verMode" == "cluster" ]; then sed -i 's/verMode=edge/verMode=cluster/g' install_client_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_client_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_client_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_client_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_client_temp.sh sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_client_temp.sh diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index a2dec155e8..8a41b13ccb 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -51,9 +51,9 @@ fi if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then cd ${top_dir}/tools/taos-tools/packaging/deb - + taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1) - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" + [ -z "$taostools_ver" ] && taostools_ver="0.1.0" taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}" cd ${curr_dir} @@ -96,7 +96,7 @@ else ${taostools_bin_files} \ ${taosx_bin} \ ${explorer_bin_files} \ - ${build_dir}/bin/taosadapter \ + ${build_dir}/bin/${clientName}adapter \ ${build_dir}/bin/udfd \ ${script_dir}/remove.sh \ ${script_dir}/set_core.sh \ @@ -135,12 +135,12 @@ mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} -if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then - cp ${compile_dir}/test/cfg/taosadapter.toml ${install_dir}/cfg || : +if [ -f "${compile_dir}/test/cfg/${clientName}adapter.toml" ]; then + cp ${compile_dir}/test/cfg/${clientName}adapter.toml ${install_dir}/cfg || : fi -if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then - cp ${compile_dir}/test/cfg/taosadapter.service ${install_dir}/cfg || : +if [ -f "${compile_dir}/test/cfg/${clientName}adapter.service" ]; then + cp ${compile_dir}/test/cfg/${clientName}adapter.service ${install_dir}/cfg || : fi if [ -f "${cfg_dir}/${serverName}.service" ]; then @@ -152,16 +152,16 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${se mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm if [ $adapterName != "taosadapter" ]; then - mv ${install_dir}/cfg/taosadapter.toml ${install_dir}/cfg/$adapterName.toml + mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml sed -i "s/path = \"\/var\/log\/taos\"/path = \"\/var\/log\/${productName}\"/g" ${install_dir}/cfg/$adapterName.toml sed -i "s/password = \"taosdata\"/password = \"${defaultPasswd}\"/g" ${install_dir}/cfg/$adapterName.toml - mv ${install_dir}/cfg/taosadapter.service ${install_dir}/cfg/$adapterName.service + mv ${install_dir}/cfg/${clientName2}adapter.service ${install_dir}/cfg/$adapterName.service sed -i "s/TDengine/${productName}/g" ${install_dir}/cfg/$adapterName.service sed -i "s/taosAdapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service sed -i "s/taosadapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service - mv ${install_dir}/bin/taosadapter ${install_dir}/bin/${adapterName} + mv ${install_dir}/bin/${clientName2}adapter ${install_dir}/bin/${adapterName} mv ${install_dir}/bin/taosd-dump-cfg.gdb ${install_dir}/bin/${serverName}-dump-cfg.gdb fi @@ -233,8 +233,10 @@ if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" remove_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" remove_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" remove_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" remove_temp.sh - sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" remove_temp.sh + cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'` + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" remove_temp.sh mv remove_temp.sh ${install_dir}/bin/remove.sh fi if [ "$verMode" == "cloud" ]; then @@ -262,8 +264,10 @@ if [ "$verMode" == "cluster" ]; then sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_temp.sh - sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_temp.sh + cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'` + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" install_temp.sh mv install_temp.sh ${install_dir}/install.sh fi if [ "$verMode" == "cloud" ]; then diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 2479e48670..8ed3bd74b9 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -40,11 +40,16 @@ serverName2="taosd" clientName2="taos" productName2="TDengine" +adapterName2="${clientName2}adapter" +demoName2="${clientName2}demo" benchmarkName2="${clientName2}Benchmark" dumpName2="${clientName2}dump" +keeperName2="${clientName2}keeper" +xName2="${clientName2}x" +explorerName2="${clientName2}-explorer" uninstallScript2="rm${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" #install main path install_main_dir=${installDir} @@ -55,8 +60,8 @@ local_bin_link_dir="/usr/local/bin" service_config_dir="/etc/systemd/system" -taos_service_name=${serverName} -taosadapter_service_name="taosadapter" +taos_service_name=${serverName2} +taosadapter_service_name="${clientName2}adapter" tarbitrator_service_name="tarbitratord" csudo="" if command -v sudo >/dev/null; then @@ -84,14 +89,14 @@ else fi function kill_taosadapter() { - pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + pid=$(ps -ef | grep "${adapterName2}" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi } function kill_taosd() { - pid=$(ps -ef | grep ${serverName} | grep -v "grep" | awk '{print $2}') + pid=$(ps -ef | grep ${serverName2} | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi @@ -109,17 +114,17 @@ function clean_bin() { ${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || : ${csudo}rm -f ${bin_link_dir}/udfd || : - ${csudo}rm -f ${bin_link_dir}/taosadapter || : - ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : - ${csudo}rm -f ${bin_link_dir}/taosdemo || : - ${csudo}rm -f ${bin_link_dir}/taosdump || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/${adapterName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/tarbitrator || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : - ${csudo}rm -f ${bin_link_dir}/taoskeeper || : - ${csudo}rm -f ${bin_link_dir}/taosx || : - ${csudo}rm -f ${bin_link_dir}/taos-explorer || : + ${csudo}rm -f ${bin_link_dir}/${keeperName2} || : + ${csudo}rm -f ${bin_link_dir}/${xName2} || : + ${csudo}rm -f ${bin_link_dir}/${explorerName2} || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : @@ -130,8 +135,8 @@ function clean_bin() { } function clean_local_bin() { - ${csudo}rm -f ${local_bin_link_dir}/taosBenchmark || : - ${csudo}rm -f ${local_bin_link_dir}/taosdemo || : + ${csudo}rm -f ${local_bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${local_bin_link_dir}/${demoName2} || : } function clean_lib() { @@ -173,7 +178,7 @@ function clean_service_on_systemd() { ${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null ${csudo}rm -f ${taosd_service_config} - taosadapter_service_config="${service_config_dir}/taosadapter.service" + taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service" if systemctl is-active --quiet ${taosadapter_service_name}; then echo "${productName2} ${clientName2}Adapter is running, stopping it..." ${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null @@ -235,8 +240,8 @@ function clean_service_on_sysvinit() { function clean_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : + ${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : } function clean_service() { diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 10a0fb5e02..2bdb56fac2 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -15,11 +15,12 @@ uninstallScript="rmtaos" clientName2="taos" productName2="TDengine" -benchmarkName2="${clientName}Benchmark" -dumpName2="${clientName}dump" -uninstallScript2="rm${clientName}" +benchmarkName2="${clientName2}Benchmark" +demoName2="${clientName2}demo" +dumpName2="${clientName2}dump" +uninstallScript2="rm${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" #install main path install_main_dir=${installDir} @@ -44,14 +45,17 @@ function kill_client() { function clean_bin() { # Remove link - ${csudo}rm -f ${bin_link_dir}/${clientName} || : - ${csudo}rm -f ${bin_link_dir}/taosdemo || : - ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : fi diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index dba1dbcf9a..418103f2a6 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -30,6 +30,10 @@ #include "tsched.h" #include "ttime.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -542,9 +546,15 @@ void taos_init_imp(void) { deltaToUtcInitOnce(); - if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + char logDirName[64] = {0}; +#ifdef CUS_PROMPT + snprintf(logDirName, 64, "%slog", CUS_PROMPT); +#else + snprintf(logDirName, 64, "taoslog"); +#endif + if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { // ignore create log failed, only print - printf(" WARING: Create taoslog failed:%s. configDir=%s\n", strerror(errno), configDir); + printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); } if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index aeeec1d61c..1c2d533977 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -228,7 +228,11 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); if (taosIsDir(cfgDir)) { +#ifdef CUS_PROMPT + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", CUS_PROMPT, cfgDir); +#else snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); +#endif } else { tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); } diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 9a092e2df5..f0e020edfe 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -20,6 +20,21 @@ #include "tglobal.h" #include "version.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#else +#ifndef CUS_NAME + #define CUS_NAME "TDengine" +#endif + +#ifndef CUS_PROMPT + #define CUS_PROMPT "taos" +#endif + +#ifndef CUS_EMAIL + #define CUS_EMAIL "" +#endif +#endif // clang-format off #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." #define DM_CFG_DIR "Configuration directory." @@ -232,7 +247,7 @@ static void dmDumpCfg() { } static int32_t dmInitLog() { - return taosCreateLog("taosdlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); + return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); } static void taosCleanupArgs() { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 52309a7b35..84004ed3c1 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -17,6 +17,10 @@ #include "os.h" #include "taoserror.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define PROCESS_ITEM 12 #define UUIDLEN37 37 @@ -252,7 +256,11 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { #ifdef WINDOWS // ASSERT(0); #elif defined(_TD_DARWIN_64) +#ifdef CUS_PROMPT + const char *filepath = "/usr/local/"CUS_PROMPT"/email"; +#else const char *filepath = "/usr/local/taos/email"; +#endif // CUS_PROMPT TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) return false; @@ -264,8 +272,12 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { taosCloseFile(&pFile); return 0; +#else +#ifdef CUS_PROMPT + const char *filepath = "/usr/local/"CUS_PROMPT"/email"; #else const char *filepath = "/usr/local/taos/email"; +#endif // CUS_PROMPT TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) return false; diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 600e33feab..7b1dc3a541 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -26,10 +26,6 @@ ELSE () SET(LINK_WEBSOCKET "") ENDIF () -IF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) - ADD_DEFINITIONS(-I${CMAKE_CURRENT_SOURCE_DIR}/../../../enterprise/packaging) -ENDIF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) - IF (TD_LINUX AND TD_ALPINE) SET(LINK_ARGP "/usr/lib/libargp.a") ELSE () diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 8111ccc0b1..0f5585991e 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -59,8 +59,8 @@ char cusEmail[] = ""; #define SHELL_VERSION "Print program version." #ifdef WEBSOCKET -#define SHELL_DSN "The dsn to use when connecting to cloud server." -#define SHELL_REST "Use restful mode when connecting." +#define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection." +#define SHELL_REST "Use RESTful mode when connecting." #define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 30." #endif @@ -93,7 +93,11 @@ void shellPrintHelp() { #endif printf("%s%s%s%s\r\n", indent, "-w,", indent, SHELL_WIDTH); printf("%s%s%s%s\r\n", indent, "-V,", indent, SHELL_VERSION); - printf("\r\n\r\nReport bugs to %s.\r\n", cusEmail); +#ifdef CUS_EMAIL + printf("\r\n\r\nReport bugs to %s.\r\n", CUS_EMAIL); +#else + printf("\r\n\r\nReport bugs to %s.\r\n", "support@taosdata.com"); +#endif } #ifdef LINUX @@ -105,7 +109,11 @@ void shellPrintHelp() { #endif const char *argp_program_version = version; -const char *argp_program_bug_address = cusEmail; +#ifdef CUS_EMAIL +const char *argp_program_bug_address = CUS_EMAIL; +#else +const char *argp_program_bug_address = "support@taosdata.com"; +#endif static struct argp_option shellOptions[] = { {"host", 'h', "HOST", 0, SHELL_HOST}, @@ -411,10 +419,19 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shell.info.clientVersion = "Welcome to the %s Command Line Interface, Client Version:%s\r\n" "Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n"; - strcpy(shell.info.cusName, cusName); - sprintf(shell.info.promptHeader, "%s> ", cusPrompt); +#ifdef CUS_NAME + strcpy(shell.info.cusName, CUS_NAME); +#else + strcpy(shell.info.cusName, "TDengine"); +#endif char promptContinueFormat[32] = {0}; - sprintf(promptContinueFormat, "%%%zus> ", strlen(cusPrompt)); +#ifdef CUS_PROMPT + sprintf(shell.info.promptHeader, "%s> ", CUS_PROMPT); + sprintf(promptContinueFormat, "%%%zus> ", strlen(CUS_PROMPT)); +#else + sprintf(shell.info.promptHeader, "taos> "); + sprintf(promptContinueFormat, "%%%zus> ", strlen("taos")); +#endif sprintf(shell.info.promptContinue, promptContinueFormat, " "); shell.info.promptSize = strlen(shell.info.promptHeader); #ifdef TD_ENTERPRISE From 1c6bf2d1d36fb912a6604e732246bdc6644edfaf Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 7 Apr 2023 18:08:49 +0800 Subject: [PATCH 169/176] fix: fix nullbitmap shift error when trimming data blocks. --- source/common/src/tdatablock.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a65f30f023..af42ca6040 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1590,12 +1590,13 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { i += 1; } } else if (n > 8) { - int32_t gap = len - newLen; + int32_t remain = (total % 8 <= tail) ? 1 : 0; + int32_t gap = len - newLen - remain; while (i < newLen) { uint8_t v = p[i + gap]; p[i] = (v << tail); - if (i < newLen - 1) { + if (i < newLen - 1 + remain) { uint8_t next = p[i + gap + 1]; p[i] |= (next >> (8 - tail)); } From f6f4e8133340ef33a53135dcca0934a6077464dc Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 7 Apr 2023 18:24:06 +0800 Subject: [PATCH 170/176] test:add testcase of childtale_from_to --- .../5-taos-tools/taosbenchmark/default_json.py | 4 ++-- .../5-taos-tools/taosbenchmark/json/default.json | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index 7599c82483..f5aa181aed 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -61,9 +61,9 @@ class TDTestCase: os.system("%s" % cmd) tdSql.execute("reset query cache") tdSql.query("show db.tables") - tdSql.checkRows(10) + tdSql.checkRows(8) tdSql.query("select count(*) from db.stb") - tdSql.checkData(0, 0, 100) + tdSql.checkData(0, 0, 80) def stop(self): tdSql.close() diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json index da22ef75e2..c841e90b51 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json @@ -15,13 +15,18 @@ "num_of_records_per_req": 10, "databases": [{ "dbinfo": { - "name": "db" + "name": "db", + "drop": "yes" + }, "super_tables": [{ + "child_table_exists":"no", "name": "stb", "childtable_prefix": "stb_", "childtable_count": 10, "insert_rows": 10, + "childtable_from": 1, + "childtable_to": 9, "columns": [{"type": "INT"}], "tags": [{"type": "INT"}] }] From af5a16b129040dd953a79ec73bcebdf80f76c317 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 7 Apr 2023 18:54:37 +0800 Subject: [PATCH 171/176] fix: set block allocated to false for each call to downstream operator --- source/libs/executor/src/aggregateoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index d5fc507b94..ec8060348d 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -172,9 +172,9 @@ int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { int32_t scanFlag = MAIN_SCAN; bool hasValidBlock = false; - bool blockAllocated = false; while (1) { + bool blockAllocated = false; SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { if (!hasValidBlock) { @@ -570,4 +570,4 @@ void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { pCtx->input.colDataSMAIsSet = pStatus->hasAgg; pCtx->input.numOfRows = pStatus->numOfRows; pCtx->input.startRowIndex = pStatus->startOffset; -} \ No newline at end of file +} From 8b063357e1b39ab44d137d94fda78ff2412219ec Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 7 Apr 2023 18:08:49 +0800 Subject: [PATCH 172/176] fix: fix nullbitmap shift error when trimming data blocks. --- tests/system-test/2-query/limit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py index c00e3b7d56..f51ed06008 100644 --- a/tests/system-test/2-query/limit.py +++ b/tests/system-test/2-query/limit.py @@ -288,7 +288,7 @@ class TDTestCase: tdSql.checkData(0, 3, 5.000000000) tdSql.checkData(0, 4, 5.000000000) tdSql.checkData(0, 5, 0.000000000) - tdSql.checkData(0, 7, 1) + tdSql.checkData(0, 7, None) tdSql.checkData(0, 8, "binary5") tdSql.checkData(0, 9, "nchar5") tdSql.checkData(1, 8, None) From 2d0dad7a37d32278fa21883fc1daa32880bb5b1d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 7 Apr 2023 18:08:49 +0800 Subject: [PATCH 173/176] fix: fix nullbitmap shift error when trimming data blocks. --- source/common/src/tdatablock.c | 2 +- tests/system-test/2-query/limit.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index af42ca6040..0dd8cb9b0c 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1590,7 +1590,7 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { i += 1; } } else if (n > 8) { - int32_t remain = (total % 8 <= tail) ? 1 : 0; + int32_t remain = (total % 8 != 0 && total % 8 <= tail) ? 1 : 0; int32_t gap = len - newLen - remain; while (i < newLen) { uint8_t v = p[i + gap]; diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py index f51ed06008..c00e3b7d56 100644 --- a/tests/system-test/2-query/limit.py +++ b/tests/system-test/2-query/limit.py @@ -288,7 +288,7 @@ class TDTestCase: tdSql.checkData(0, 3, 5.000000000) tdSql.checkData(0, 4, 5.000000000) tdSql.checkData(0, 5, 0.000000000) - tdSql.checkData(0, 7, None) + tdSql.checkData(0, 7, 1) tdSql.checkData(0, 8, "binary5") tdSql.checkData(0, 9, "nchar5") tdSql.checkData(1, 8, None) From 7dbe4f6ca1c15b64c2da742d8eb83fe1d0251934 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 8 Apr 2023 00:50:43 +0800 Subject: [PATCH 174/176] fix: cfg name release script community main (#20818) * chore: cus name support in script * chore: merge with 3.0 * fix: source/common/src/tglobal.c * fix: source/common/src/tglobal.c --- source/common/src/tglobal.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 1c2d533977..56e34da4ce 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -20,6 +20,10 @@ #include "tlog.h" #include "tmisce.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + GRANT_CFG_DECLARE; SConfig *tsCfg = NULL; @@ -229,7 +233,7 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); if (taosIsDir(cfgDir)) { #ifdef CUS_PROMPT - snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", CUS_PROMPT, cfgDir); + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", cfgDir, CUS_PROMPT); #else snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); #endif From f89301b5cd24a4c651b57a5f9828c44d00f9f997 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 9 Apr 2023 10:51:38 +0800 Subject: [PATCH 175/176] fix: taosdump in diff type for main (#20825) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index b427177e5b..0110b27b32 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 63635fc + GIT_TAG 149ac34 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 015c81cdb02f0185b766c2839e01a6389fd8efcc Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Sun, 9 Apr 2023 20:43:18 +0800 Subject: [PATCH 176/176] feat(release): install taosx and taos-explorer service for enterprise --- packaging/tools/install.sh | 36 ++++++++++++++++++++++++++++++++++++ packaging/tools/makepkg.sh | 1 + packaging/tools/remove.sh | 22 +++++++++++++++++++++- 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 858a6ac668..63ea55cf82 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -572,6 +572,20 @@ function install_config() { done } +function install_share_etc() { + for c in `ls ${script_dir}/share/etc/`; do + if [ -e /etc/$c ]; then + out=/etc/$c.new.`date +%F` + echo -e -n "${RED} /etc/$c exists, save a new cfg file as $out" + ${csudo}cp -f ${script_dir}/share/etc/$c $out + else + ${csudo}cp -f ${script_dir}/share/etc/$c /etc/$c + fi + done + + ${csudo} cp ${script_dir}/share/srv/* ${service_config_dir} +} + function install_log() { ${csudo}rm -rf ${log_dir} || : ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} @@ -685,11 +699,33 @@ function clean_service_on_systemd() { # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then # ${csudo}rm -f ${service_config_dir}/${serverName2}.service # fi + x_service_config="${service_config_dir}/${xName2}.service" + if [ -e "$x_service_config" ]; then + if systemctl is-active --quiet ${xName2}; then + echo "${productName2} ${xName2} is running, stopping it..." + ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${x_service_config} + fi + + explorer_service_config="${service_config_dir}/${explorerName2}.service" + if [ -e "$explorer_service_config" ]; then + if systemctl is-active --quiet ${explorerName2}; then + echo "${productName2} ${explorerName2} is running, stopping it..." + ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${explorer_service_config} + ${csudo}rm -f /etc/${clientName2}/explorer.toml + fi } function install_service_on_systemd() { clean_service_on_systemd + install_share_etc + [ -f ${script_dir}/cfg/${serverName2}.service ] && ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ ${service_config_dir}/ || : diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 8a41b13ccb..12e215c62b 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -150,6 +150,7 @@ fi mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm +mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share if [ $adapterName != "taosadapter" ]; then mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 8ed3bd74b9..6c671473bf 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -192,7 +192,27 @@ function clean_service_on_systemd() { ${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${tarbitratord_service_config} + + x_service_config="${service_config_dir}/${xName2}.service" + if [ -e "$x_service_config" ]; then + if systemctl is-active --quiet ${xName2}; then + echo "${productName2} ${xName2} is running, stopping it..." + ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${x_service_config} + fi + + explorer_service_config="${service_config_dir}/${explorerName2}.service" + if [ -e "$explorer_service_config" ]; then + if systemctl is-active --quiet ${explorerName2}; then + echo "${productName2} ${explorerName2} is running, stopping it..." + ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${explorer_service_config} + ${csudo}rm -f /etc/${clientName2}/explorer.toml + fi } function clean_service_on_sysvinit() {