From 0d9d2ea293c5b0679be29bdafa7f2ed5eab1188c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Feb 2023 18:31:32 +0800 Subject: [PATCH 001/187] fix(query): do some internal refactor, and fix the bug when calender time duration exists in sliding and offset. --- include/common/ttime.h | 2 +- include/libs/executor/executor.h | 4 +- source/common/src/ttime.c | 114 +++++++++++------- source/libs/executor/inc/executil.h | 3 + source/libs/executor/inc/executorimpl.h | 1 - source/libs/executor/src/executil.c | 37 +++++- source/libs/executor/src/executorimpl.c | 6 +- source/libs/executor/src/filloperator.c | 8 +- source/libs/executor/src/scanoperator.c | 37 +----- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 47 +------- source/libs/planner/src/planOptimizer.c | 4 +- 12 files changed, 129 insertions(+), 136 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index eaf44c2771..69a071d59f 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -74,7 +74,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 63e1c556de..382837951a 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -178,7 +178,9 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key); /** * return the scan info, in the form of tuple of two items, including table uid and current timestamp * @param tinfo diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 559ffd2aaf..bbb58cc63d 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -82,6 +82,7 @@ static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_ static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); +static int32_t parseTimezone(char* str, int64_t* tzOffset); static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = { parseLocaltime, parseLocaltimeDst}; @@ -708,21 +709,19 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati return getDuration(*duration, *unit, duration, timePrecision); } +#define IS_CALENDAR_TIME_DURATION(_d) (((_d) == 'n') || ((_d) == 'y')) + int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { return t; } - if (unit != 'n' && unit != 'y') { + if (!IS_CALENDAR_TIME_DURATION(unit)) { return t + duration; } // The following code handles the y/n time duration - int64_t numOfMonth = duration; - if (unit == 'y') { - numOfMonth *= 12; - } - + int64_t numOfMonth = (unit == 'y')? duration*12:duration; int64_t fraction = t % TSDB_TICK_PER_SECOND(precision); struct tm tm; @@ -764,13 +763,16 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char return (emon - smon) / (int32_t)interval; } -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (pInterval->sliding == 0 && pInterval->interval == 0) { - return t; + return ts; } - int64_t start = t; - if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') { + int64_t start = ts; + int32_t precision = pInterval->precision; + + if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { + start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; @@ -792,44 +794,72 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start = (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision)); } else { - int64_t delta = t - pInterval->interval; - int32_t factor = (delta >= 0) ? 1 : -1; + if (IS_CALENDAR_TIME_DURATION(pInterval->intervalUnit)) { + int64_t news = (ts / pInterval->sliding) * pInterval->sliding; + ASSERT(news <= ts); - start = (delta / pInterval->sliding + factor) * pInterval->sliding; + if (news <= ts) { + int64_t prev = news; + int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; - if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { - /* - * here we revised the start time of day according to the local time zone, - * but in case of DST, the start time of one day need to be dynamically decided. - */ - // todo refactor to extract function that is available for Linux/Windows/Mac platform -#if defined(WINDOWS) && _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char** tzname = _tzname; -#endif + if (newe < ts) { // move towards the greater endpoint + while(newe < ts && news < ts) { + news += pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } - start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); - } - - int64_t end = 0; - - // not enough time range - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - while (end < t) { // move forward to the correct time window - start += pInterval->sliding; - - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = start + pInterval->interval - 1; + prev = news; } else { - end = INT64_MAX; - break; + while (newe >= ts) { + prev = news; + news -= pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } } + + return prev; } } else { - end = INT64_MAX; + int64_t delta = ts - pInterval->interval; + int32_t factor = (delta >= 0) ? 1 : -1; + + start = (delta / pInterval->sliding + factor) * pInterval->sliding; + + if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { + /* + * here we revised the start time of day according to the local time zone, + * but in case of DST, the start time of one day need to be dynamically decided. + */ + // todo refactor to extract function that is available for Linux/Windows/Mac platform +#if defined(WINDOWS) && _MSC_VER >= 1900 + // see + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char** tzname = _tzname; +#endif + + start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); + } + + int64_t end = 0; + + // not enough time range + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + while (end < ts) { // move forward to the correct time window + start += pInterval->sliding; + + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = start + pInterval->interval - 1; + } else { + end = INT64_MAX; + break; + } + } + } else { + end = INT64_MAX; + } } } @@ -842,7 +872,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; int64_t newEnd = end; - while (newEnd >= t) { + while (newEnd >= ts) { end = newEnd; newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f99c7de93d..20f43deaa3 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -164,4 +164,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, void printDataBlock(SSDataBlock* pBlock, const char* flag); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 999a7965fb..1850cd007f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -854,7 +854,6 @@ int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResul SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 040e67713d..53947e85e7 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1686,12 +1686,12 @@ int32_t convertFillType(int32_t mode) { return type; } -static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { if (ascQuery) { - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); int64_t key = w->skey; while (key < ts) { // moving towards end @@ -1708,7 +1708,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindo static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { STimeWindow w = {0}; - w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision); + w.skey = taosTimeTruncate(ts, pInterval); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return w; } @@ -1742,6 +1742,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI if (pRow) { w = pRow->win; } + // in case of typical time window, we can calculate time window directly. if (w.skey > ts || w.ekey < ts) { w = doCalculateTimeWindow(ts, pInterval); @@ -1756,6 +1757,34 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI return w; } +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (pInterval->sliding != 'n' && pInterval->sliding != 'y') { + tw->skey += pInterval->sliding * factor; + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return; + } + + // convert key to second + int64_t key = convertTimePrecision(tw->skey, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + int64_t duration = pInterval->sliding; + if (pInterval->slidingUnit == 'y') { + duration *= 12; + } + + struct tm tm; + time_t t = (time_t) key; + taosLocalTime(&t, &tm); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + duration * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); + + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; +} + bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { return (pLimitInfo->limit.limit != -1 || pLimitInfo->limit.offset != -1 || pLimitInfo->slimit.limit != -1 || pLimitInfo->slimit.offset != -1); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d28c3cfe58..464282613d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -637,15 +637,15 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code) ? tr void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; } ///////////////////////////////////////////////////////////////////////////////////////////// -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) { +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) { STimeWindow win = {0}; - win.skey = taosTimeTruncate(key, pInterval, precision); + win.skey = taosTimeTruncate(key, pInterval); /* * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; if (win.ekey < win.skey) { win.ekey = INT64_MAX; } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 16983cb507..3403c202b7 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -255,10 +255,10 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); + STimeWindow w = {0}; int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey); - w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order); + getInitialStartTimeWindow(pInterval, startKey, &w, order); pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); @@ -400,13 +400,13 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); return win.skey; } TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); return win.skey; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b556733254..c83c161326 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -67,39 +67,6 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - tw->ekey -= 1; -} - static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) { STimeWindow w = {0}; @@ -109,7 +76,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } if (order == TSDB_ORDER_ASC) { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey); ASSERT(w.ekey >= pBlockInfo->window.skey); if (w.ekey < pBlockInfo->window.ekey) { @@ -128,7 +95,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } } } else { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey); ASSERT(w.skey <= pBlockInfo->window.ekey); if (w.skey > pBlockInfo->window.skey) { diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 778281f9b4..ed786003d5 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -519,7 +519,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) pFillInfo->end = endKey; if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->interval.precision); + pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval); } pFillInfo->index = 0; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c5dc927bd1..15f803125c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -263,43 +263,6 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary return num; } -static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - tw->ekey -= 1; -} - -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - getNextTimeWindow(pInterval, pInterval->precision, order, tw); -} - void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { SqlFunctionCtx* pCtx = pSup->pCtx; @@ -455,7 +418,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; - getNextTimeWindow(pInterval, precision, order, pNext); + getNextTimeWindow(pInterval, pNext, order); // next time window is not in current block if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || @@ -500,7 +463,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, if (ascQuery && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -509,7 +472,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -1380,7 +1343,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa do { if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i])) { - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } uint64_t winGpId = pGpDatas[i]; @@ -1397,7 +1360,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); } } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); } while (win.ekey <= endTsCols[i]); } } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 90a7261074..27c1555666 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1247,8 +1247,8 @@ static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pW .sliding = pIndex->sliding, .slidingUnit = pIndex->slidingUnit, .precision = pScan->node.precision}; - return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval, pScan->node.precision)) && - (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval, pScan->node.precision)); + return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval)) && + (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval)); } return true; } From 9b90416bf32effdd2617124394cedeab9dd2b721 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Feb 2023 18:38:08 +0800 Subject: [PATCH 002/187] test: add unit test case. --- source/libs/executor/test/timewindowTest.cpp | 141 +++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 source/libs/executor/test/timewindowTest.cpp diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp new file mode 100644 index 0000000000..6635c820fa --- /dev/null +++ b/source/libs/executor/test/timewindowTest.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taos.h" +#include "thash.h" +#include "tsimplehash.h" +#include "executor.h" +#include "ttime.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +namespace { +SInterval createInterval(int64_t interval, int64_t sliding, int64_t offset, char intervalUnit, char slidingUnit, + char offsetUnit, int8_t precision) { + SInterval v = {0}; + v.interval = interval; + v.intervalUnit = intervalUnit; + v.sliding = sliding; + v.slidingUnit = slidingUnit; + v.offset = offset; + v.offsetUnit = offsetUnit; + v.precision = precision; + return v; +} + +void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) { + char buf[64] = {0}; + char bufs[64] = {0}; + char bufe[64] = {0}; + + taosFormatUtcTime(buf, tListLen(buf), ts, precision); + + taosFormatUtcTime(bufs, tListLen(bufs), pWindow->skey, precision); + taosFormatUtcTime(bufe, tListLen(bufe), pWindow->ekey, precision); + + printf("%s [%s - %s]\n", buf, bufs, bufe); +} +} // namespace + +TEST(testCase, timewindow_gen) { + // time window + osSetTimezone("UTC"); + + SInterval interval = + createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + + int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms) + + STimeWindow w = {0}; + getInitialStartTimeWindow(&interval, key, &w, true); + printTimeWindow(&w, interval.precision, key); + + getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, interval.precision, key); + + printf("---------------------------------------------------\n"); + SInterval monthInterval = + createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&monthInterval, key, &w, true); + printTimeWindow(&w, monthInterval.precision, key); + + getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, monthInterval.precision, key); + + printf("----------------------------------------------------------\n"); + SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&slidingInterval, key, &w, true); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + printf("----------------------------------------------------------\n"); + SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + int64_t k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true); + printTimeWindow(&w, calendar_interval_1n.precision, k1); + + printf("next\n"); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + + printf("----------------------------------------------------------\n"); + SInterval calendar_interval_1d_sliding_1n = createInterval(1*86400*100L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + +} + +#pragma GCC diagnostic pop \ No newline at end of file From 579b1b0a6c142c26b6f14261076d7ec54aee7bd1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 Feb 2023 15:12:19 +0800 Subject: [PATCH 003/187] fix(query): check correct unit. --- source/libs/executor/src/executil.c | 2 +- source/libs/executor/test/timewindowTest.cpp | 65 ++++++++++++-------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 53947e85e7..91f384ce6a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1759,7 +1759,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->sliding != 'n' && pInterval->sliding != 'y') { + if (pInterval->slidingUnit != 'n' && pInterval->slidingUnit != 'y') { tw->skey += pInterval->sliding * factor; tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return; diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index 6635c820fa..be5e217ed1 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -56,85 +56,102 @@ void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) { } // namespace TEST(testCase, timewindow_gen) { - // time window + // set correct time zone osSetTimezone("UTC"); - + int32_t precision = TSDB_TIME_PRECISION_MILLI; + SInterval interval = - createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', precision); int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms) STimeWindow w = {0}; getInitialStartTimeWindow(&interval, key, &w, true); - printTimeWindow(&w, interval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC); printf("next\n"); - printTimeWindow(&w, interval.precision, key); + printTimeWindow(&w, precision, key); printf("---------------------------------------------------\n"); SInterval monthInterval = createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI); getInitialStartTimeWindow(&monthInterval, key, &w, true); - printTimeWindow(&w, monthInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC); printf("next\n"); - printTimeWindow(&w, monthInterval.precision, key); + printTimeWindow(&w, precision, key); printf("----------------------------------------------------------\n"); SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); getInitialStartTimeWindow(&slidingInterval, key, &w, true); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); printf("next\n"); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); - printf("----------------------------------------------------------\n"); + printf("-----------------calendar_interval_1n_sliding_1d-------\n"); SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); int64_t k1 = 1664409600 * 1000L; getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true); - printTimeWindow(&w, calendar_interval_1n.precision, k1); + printTimeWindow(&w, precision, k1); printf("next\n"); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); + printf("----------------interval_1d_clendar_sliding_1n---------\n"); + SInterval interval_1d_calendar_sliding_1n = createInterval(1*86400*1000L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); - printf("----------------------------------------------------------\n"); - SInterval calendar_interval_1d_sliding_1n = createInterval(1*86400*100L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&interval_1d_calendar_sliding_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + printf("next time window:\n"); + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n"); + SInterval offset_1n = createInterval(1*86400*1000L, 1*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); } From a488f31ba98998cd338e9f21dd135999e78a332f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Feb 2023 09:52:11 +0800 Subject: [PATCH 004/187] refactor:do some internal refactor. --- source/common/src/ttime.c | 8 ++++---- source/libs/executor/test/timewindowTest.cpp | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index bbb58cc63d..de6a18f7c8 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -871,10 +871,10 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { // try to move current window to the left-hande-side, due to the offset effect. int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - int64_t newEnd = end; - while (newEnd >= ts) { - end = newEnd; - newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); + int64_t newe = end; + while (newe >= ts) { + end = newe; + newe = taosTimeAdd(newe, -pInterval->sliding, pInterval->slidingUnit, precision); } start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index be5e217ed1..2894c66587 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -151,7 +151,10 @@ TEST(testCase, timewindow_gen) { printTimeWindow(&w, precision, k1); printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n"); - SInterval offset_1n = createInterval(1*86400*1000L, 1*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); + SInterval offset_1n = createInterval(10*86400*1000L, 10*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&offset_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + } From 2400104ec80fc98ac91a8c8d8253374565601cfe Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 11 May 2023 14:26:42 +0800 Subject: [PATCH 005/187] test: init add eco-system --- tests/system-test/eco-system/main.py | 37 +++ .../eco-system/manager/drop_table.py | 146 +++++++++++ .../eco-system/manager/same_column.py | 181 +++++++++++++ .../eco-system/manager/schema_change.py | 239 ++++++++++++++++++ .../eco-system/schemaless/insert.py | 147 +++++++++++ 5 files changed, 750 insertions(+) create mode 100644 tests/system-test/eco-system/main.py create mode 100644 tests/system-test/eco-system/manager/drop_table.py create mode 100644 tests/system-test/eco-system/manager/same_column.py create mode 100644 tests/system-test/eco-system/manager/schema_change.py create mode 100644 tests/system-test/eco-system/schemaless/insert.py diff --git a/tests/system-test/eco-system/main.py b/tests/system-test/eco-system/main.py new file mode 100644 index 0000000000..a1f72147a0 --- /dev/null +++ b/tests/system-test/eco-system/main.py @@ -0,0 +1,37 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + def run(self): + tdLog.info(" ------ eco-system main -------") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/eco-system/manager/drop_table.py b/tests/system-test/eco-system/manager/drop_table.py new file mode 100644 index 0000000000..d4cdf4a541 --- /dev/null +++ b/tests/system-test/eco-system/manager/drop_table.py @@ -0,0 +1,146 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(0, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + # show tables + def show_tables(self): + sql = "show tables;" + row_cnt = tdSql.query(sql) + tables = [] + for i in range(0, row_cnt): + tb_name = tdSql.queryResult[i][0] + tables.append(tb_name) + + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def drop_table(self, change_cnt): + # init + + tables = self.show_tables() + + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.drop_table(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/same_column.py b/tests/system-test/eco-system/manager/same_column.py new file mode 100644 index 0000000000..beaf4e449e --- /dev/null +++ b/tests/system-test/eco-system/manager/same_column.py @@ -0,0 +1,181 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'bc': 'bool', + 'fc': 'float', + 'dc': 'double', + 'ti': 'tinyint', + 'si': 'smallint', + 'ic': 'int', + 'bi': 'bigint', + 'uit': 'tinyint unsigned', + 'usi': 'smallint unsigned', + 'ui': 'int unsigned', + 'ubi': 'bigint unsigned', + 'bin': 'binary(32)', + 'nch': 'nchar(64)' + } + self.tag_dict = { + 'groupid': 'tinyint', + 'location': 'binary(16)', + 'tfc': 'float', + 'tdc': 'double', + 'tti': 'tinyint', + 'tsi': 'smallint', + 'tic': 'int', + 'tbi': 'bigint', + 'tuit': 'tinyint unsigned', + 'tusi': 'smallint unsigned', + 'tui': 'int unsigned', + 'tubi': 'bigint unsigned', + 'tbin': 'binary(32)', + 'tnch': 'nchar(64)' + } + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_columns(self, change_cnt): + # init + + ncol = len(self.column_dict) + ntag = len(self.tag_dict) + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_columns(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/schema_change.py b/tests/system-test/eco-system/manager/schema_change.py new file mode 100644 index 0000000000..400d2b100b --- /dev/null +++ b/tests/system-test/eco-system/manager/schema_change.py @@ -0,0 +1,239 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'col0': 'int', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'varchar(20)', + 'col13': 'nchar(20)' + } + self.tag_dict = { + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': 'varchar(20)', + 't13': 'nchar(20)', + 't14': 'timestamp' + } + + + # delete + def delete_col(self, columns, cnt, max_col): + # delte for random + for i in range(cnt): + col_cnt = len(columns) + if col_cnt == 0: + return + sel = random.randint(0, col_cnt - 1) + sql = f"alter table meters drop column `{columns[sel]}`" + try: + tdSql.execute(sql) + tdLog.info(f" drop cur col={len(columns)} max_col={max_col} {sql}") + del columns[sel] + except: + tdLog.info(f" drop column failed. {sql}") + traceback.print_exc() + + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def renames(self, tags, cnt): + col_cnt = len(tags) + if col_cnt < 10: + return + for i in range(cnt): + sel = random.randint(1, col_cnt-3) + new_name = tags[sel] + "n" + sql = f"alter table meters rename tag `{tags[sel]}` `{new_name}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + tags[sel] = new_name + except: + tdLog.info(f" rename tag failed. {sql}") + traceback.print_exc() + + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # query + def query_table(self, columns, tags): + if len(columns) < 10 : + return + if len(tags) < 10: + return + + sel_cols = random.sample(columns, random.randint(2,int(len(columns)/10))) + sel_tags = random.sample(tags, random.randint(1,int(len(tags)/10))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_schema(self, change_cnt): + # init + columns, tags = self.describe_table("meters") + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + for i in range(change_cnt): + col_cnt = len(self.column_dict) + icol = random.randint(0, col_cnt-1) + key = f"col{icol}" + col_name = key + f"_{i}_{random.randint(1,100)}" + col_type = self.column_dict[key] + sql = f'alter table meters add column `{col_name}` {col_type}' + sql_tag = f'alter table meters add tag `t_{col_name}` {col_type}' + + try: + tdSql.execute(sql) + tdLog.info(f" add cur col={len(columns)} max_col={max_col} {sql}") + columns.append(col_name) + if random.randint(1, 4) == 2: + tdSql.execute(sql_tag) + tdLog.info(f" add tag tag_cnt={len(tags)} {sql_tag}") + + except: + tdLog.info(f" add column failed. {sql}") + traceback.print_exc() + + + col_cnt = len(columns) + # delete + if col_cnt > max_col + 100: + self.delete_col(columns, random.randint(1, 30), max_col) + elif col_cnt >= max_col + 30: + self.delete_col(columns, random.randint(1, 4), max_col) + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + elif col_cnt > max_col: + self.delete_col(columns, random.randint(1, 3), max_col) + + + + if i % 50 == 0: + sql = f"flush database {self.dbname};" + tdSql.execute(sql) + tdLog.info(f" ***** {sql} *****") + + # query + if i % 70 == 0: + self.query_table(columns, tags) + + # drop and rename + if i % 10 == 0: + # update columns + columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(columns)} tags= {len(tags)}======") + + if random.randint(1,3) == 2: + self.query_table(columns, tags) + + if len(tags) > 50: + self.drop_tag(tags, random.randint(1, 30)) + + self.renames(tags, random.randint(1, 10)) + + + # sleep + #time.sleep(0.3) + + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_schema(1000000) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/schemaless/insert.py b/tests/system-test/eco-system/schemaless/insert.py new file mode 100644 index 0000000000..def70748d7 --- /dev/null +++ b/tests/system-test/eco-system/schemaless/insert.py @@ -0,0 +1,147 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback +import taos +import string +from taos import schemaless + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.conn = conn + self.schema = {} + + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + def genCol(self, col_name, isTag): + col_types = ["str","f64","f32","i8","u8","i16","u16","i32","u32","i64","u64"] + if self.schema.get(col_name) == None: + col_type = random.choice(col_types) + self.schema[col_name] = col_type + else: + col_type = self.schema[col_name] + + is_num = True + val = "" + if col_type == "str": + val = self.random_string(random.randint(1, 10)) + is_num = False + elif col_type == "f64": + val = random.randrange(-100000000000000, 1000000000000)/3*2.25678 + elif col_type == "f32": + val = random.randrange(-100000000, 1000000000)/3*1.2345 + elif col_type == "i8": + val = random.randint(-128, 127) + elif col_type == "u8": + val = random.randint(0, 256) + elif col_type == "i16": + val = random.randint(-32768, 32767) + elif col_type == "u16": + val = random.randint(0, 256*256) + elif col_type == "i32": + val = random.randint(-256*256*256*128, 256*256*256*128) + elif col_type == "u32": + val = random.randint(0, 256*256*256*256) + elif col_type == "i64": + val = random.randint(-256*256*256*256*256*256*256*128, 256*256*256*256*256*256*256*128) + elif col_type == "u64": + val = random.randint(0, 256*256*256*256*256*256*256*256) + else: + val = 100 + + if isTag: + col_val = val + elif is_num: + col_val = f'{val}{col_type}' + else: + col_val = '"' + val + '"' + + return f'{col_name}={col_val}' + + + # cols + def genCols(self, pre, max, index, isTag): + col_cnt = random.randint(1, max) + cols = [] + for i in range(col_cnt): + col_name = f'{pre}_{index}_{i}' + cols.append(self.genCol(col_name, isTag)) + + return ",".join(cols) + + + # execute sql + def insert(self,sql,i): + print("schema less insert") + try: + self.conn.schemaless_insert([sql], schemaless.SmlProtocol.LINE_PROTOCOL, schemaless.SmlPrecision.MILLI_SECONDS) + tdLog.info(f" exec ok i={i} {sql}") + except: + tdLog.info(f" exe failed. i={i} {sql}") + traceback.print_exc() + + + # change table schema + def schemaless_insert(self, change_cnt): + # init + ts = 1683194263000 + for i in range(change_cnt): + t1 = i % 1000 + index = int(i/10000) % 600 + cols = self.genCols("c", 5, index, False) + tags = f"t1={t1},t2=abc,t3=work" + sql = f'sta,{tags} {cols} {ts + i}' + self.insert(sql, i) + + # run + def run(self): + # seed + #random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + tdSql.execute(f"drop table if exists sta;") + + + + # change meters + try: + self.schemaless_insert(1000) + except: + traceback.print_exc() + + print(self.schema) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 629ab2b85fb59eeebc3e4634de1db0c9b0b91586 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 15 May 2023 16:10:11 +0800 Subject: [PATCH 006/187] fix: extract tag equal condition --- include/libs/nodes/plannodes.h | 2 + source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 18 ++++-- source/libs/nodes/src/nodesMsgFuncs.c | 10 ++- source/libs/nodes/src/nodesUtilFuncs.c | 2 + source/libs/planner/src/planOptimizer.c | 73 ++++++++++++++++++++++ source/libs/planner/src/planPhysiCreater.c | 3 + 7 files changed, 103 insertions(+), 6 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index ad4b59714c..0e235191b1 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,6 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; + SNode* pTagEqualConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -405,6 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; + SNode* pTagEqualCondtions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index d9a4c5178f..91db527a02 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,6 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); + CLONE_NODE_FIELD(pTagEqualConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0aeb83ce08..b4e2d95e26 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1416,6 +1416,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanMergeCondition = "MergeConditions"; +static const char* jkJoinLogicPlanTagEqualConditions = "TagEqualConditions"; static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; @@ -1430,7 +1431,9 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pTagEqualConditions); + } return code; } @@ -1447,7 +1450,9 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pTagEqualConditions); + } return code; } @@ -1878,6 +1883,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition"; static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; +static const char* jkJoinPhysiPlanTagEqualConditions = "TagEqualConditions"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; @@ -1898,7 +1904,9 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pTagEqualCondtions); + } return code; } @@ -1921,7 +1929,9 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pTagEqualCondtions); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 6c6b6c0e81..45cebb4559 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2317,7 +2317,8 @@ enum { PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, PHY_SORT_MERGE_JOIN_CODE_TARGETS, - PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER + PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, + PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS }; static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2339,7 +2340,9 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pTagEqualCondtions); + } return code; } @@ -2368,6 +2371,9 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER: code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; + case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagEqualCondtions); + break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f71eef7969..d97b17e30c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1072,6 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); + nodesDestroyNode(pLogicNode->pTagEqualConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1204,6 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); + nodesDestroyNode(pPhyNode->pTagEqualCondtions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 66d85a9c89..f5ef02e207 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -740,6 +740,75 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin return code; } +static bool pushDownCondOptIsTag(SNode* pNode, SNodeList* pTableCols) { + if (QUERY_NODE_COLUMN != nodeType(pNode)) { + return false; + } + SColumnNode* pCol = (SColumnNode*)pNode; + if (COLUMN_TYPE_TAG != pCol->colType) { + return false; + } + return pushDownCondOptBelongThisTable(pNode, pTableCols); +} + +static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { + if (QUERY_NODE_OPERATOR != nodeType(pCond)) { + return false; + } + SOperatorNode* pOper = (SOperatorNode*)pCond; + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + if (pushDownCondOptIsTag(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsTag(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsTag(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsTag(pOper->pRight, pLeftCols); + } + return false; +} + +static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); + + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pTagEqualConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (pushDownCondOptIsTagEqualCond(pJoin, pCond)) { + code = nodesListMakeAppend(&pTagEqualConds, nodesCloneNode(pCond)); + } + } + + SNode* pTempTagEqCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempTagEqCond, &pTagEqualConds); + } + + if (TSDB_CODE_SUCCESS == code) { + pJoin->pTagEqualConditions = pTempTagEqCond; + return TSDB_CODE_SUCCESS; + } else { + nodesDestroyList(pTagEqualConds); + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t pushDownCondOptJoinExtractTagEqualCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { + return pushDownCondOptJoinExtractTagEqualLogicCond(pJoin); + } + + if (pushDownCondOptIsTagEqualCond(pJoin, pJoin->pOnConditions)) { + pJoin->pTagEqualConditions = nodesCloneNode(pJoin->pOnConditions); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { return TSDB_CODE_SUCCESS; @@ -774,6 +843,10 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin); } + if (TSDB_CODE_SUCCESS == code) { + code = pushDownCondOptJoinExtractTagEqualCond(pCxt, pJoin); + } + if (TSDB_CODE_SUCCESS == code) { OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e2c2e4c655..9c6c227480 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,6 +705,9 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pTagEqualConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pTagEqualConditions, &pJoin->pTagEqualCondtions); + } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); } From 789b0c7178e5ce21ea5e4231952066d9562c6023 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 16 May 2023 15:42:30 +0800 Subject: [PATCH 007/187] fix: select , from ... return error data when strlen(expr1) > TSDB_COL_NAME_LEN --- source/libs/nodes/src/nodesToSQLFuncs.c | 4 +-- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/src/parAstCreater.c | 15 +++++++++-- source/libs/parser/src/parTranslater.c | 4 ++- source/libs/planner/src/planLogicCreater.c | 1 + tests/parallel_test/cases.task | 1 + tests/script/tsim/query/bug3398.sim | 30 ++++++++++++++++++++++ 7 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 tests/script/tsim/query/bug3398.sim diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 0181da92a9..b57bba0cc9 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -120,9 +120,9 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } if (colNode->tableAlias[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->node.userAlias); } else { - *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->node.userAlias); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f71eef7969..acd028d8ae 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2054,7 +2054,7 @@ char* nodesGetNameFromColumnNode(SNode* pNode) { return "NULL"; } - return ((SColumnNode*)pNode)->colName; + return ((SColumnNode*)pNode)->node.userAlias; } int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots) { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5a47ed731d..6f866c027e 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -259,8 +259,19 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); - strncpy(pExpr->aliasName, pRawExpr->p, len); - pExpr->aliasName[len] = '\0'; + + // See TS-3398. + // Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN]. + // If aliasName is truncated, hash value of aliasName could be the same. + T_MD5_CTX ctx; + tMD5Init(&ctx); + tMD5Update(&ctx, (uint8_t*)pRawExpr->p, pRawExpr->n); + tMD5Final(&ctx); + char* p = pExpr->aliasName; + for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { + sprintf(p, "%02x", ctx.digest[i]); + p += 2; + } strncpy(pExpr->userAlias, pRawExpr->p, len); pExpr->userAlias[len] = '\0'; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f4c86d4849..9a45add98f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -827,7 +827,7 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum strcpy(pCol->node.aliasName, pCol->colName); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pCol->colName); + strcpy(pCol->node.userAlias, pExpr->userAlias); } pCol->node.resType = pExpr->resType; } @@ -1760,6 +1760,7 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName); + strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias); pVal->node.resType = ((SExprNode*)*pNode)->resType; if (NULL == pLiteral) { pVal->isNull = true; @@ -2739,6 +2740,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { } else { len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1)); + len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); strncpy(pFunc->node.userAlias, buf, TMIN(len, sizeof(pFunc->node.userAlias) - 1)); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 39783868b3..b9ea5ba0cd 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -100,6 +100,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); + strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol); diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4065ac5bee..484ac82acd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -904,6 +904,7 @@ ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/nullColSma.sim +,,y,script,./test.sh -f tsim/query/bug3398.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim ,,y,script,./test.sh -f tsim/mnode/basic1.sim diff --git a/tests/script/tsim/query/bug3398.sim b/tests/script/tsim/query/bug3398.sim new file mode 100644 index 0000000000..3ca88cf459 --- /dev/null +++ b/tests/script/tsim/query/bug3398.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database +sql create database test + +print =============== create super table and child table +sql use test + +sql CREATE STABLE st (day timestamp, c2 int) TAGS (vin binary(32)) + +sql insert into test.g using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.t using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.tg using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) + +sql select sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end), sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end) from test.t t, test.g g, test.tg tg where t.day = g.day and t.day = tg.day and t.day between '2021-05-03' and '2023-05-04' and t.vin = 'TAG1' and t.vin = g.vin and t.vin = tg.vin group by t.day; + +print $rows $data00 $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi + +if $data01 != 3.000000000 then + return -1 +endi From c6555c3b3778808e910a2044714913cd62895d61 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 16 May 2023 16:25:55 +0800 Subject: [PATCH 008/187] fix: join operator finished --- source/libs/executor/src/joinoperator.c | 369 +++++++++++++++++++----- source/libs/planner/src/planOptimizer.c | 8 + 2 files changed, 307 insertions(+), 70 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 754b5f4737..a5591b22b3 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -30,11 +30,11 @@ typedef struct SJoinRowCtx { bool rowRemains; int64_t ts; SArray* leftRowLocations; - SArray* rightRowLocations; SArray* leftCreatedBlocks; SArray* rightCreatedBlocks; int32_t leftRowIdx; int32_t rightRowIdx; + SSHashObj* buildTableTSRange; } SJoinRowCtx; typedef struct SJoinOperatorInfo { @@ -50,6 +50,17 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; + SNode* pTagEqualConditions; + + SArray* leftTagCols; + SArray* leftTagKeys; + char* leftTagKeyBuf; + int32_t leftTagKeyLen; + + SArray* rightTagCols; + SArray* rightTagKeys; + char* rightTagKeyBuf; + int32_t rightTagKeyLen; SJoinRowCtx rowCtx; } SJoinOperatorInfo; @@ -92,6 +103,147 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } +static void extractTagEqualColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, + SColumn* pLeft, SColumn* pRight) { + SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft; + SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight; + if (pLeftNode->dataBlockId == pRightNode->dataBlockId || pLeftNode->dataBlockId == pDownstreams[0]->resultDataBlockId) { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + } else { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + } +} + +static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pTagEqualNode, + SArray* leftTagEqCols, SArray* rightTagEqCols) { + SColumn left = {0}; + SColumn right = {0}; + if (nodeType(pTagEqualNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pTagEqualNode)->condType == LOGIC_COND_TYPE_AND) { + SNode* pNode = NULL; + FOREACH(pNode, ((SLogicConditionNode*)pTagEqualNode)->pParameterList) { + SOperatorNode* pOperNode = (SOperatorNode*)pNode; + extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } + return; + } + + if (nodeType(pTagEqualNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pOperNode = (SOperatorNode*)pTagEqualNode; + extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } +} + +static int32_t initTagColsGroupkeys(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { + *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); + if ((*pGroupColVals) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); + (*keyLen) += pCol->bytes; // actual data + null_flag + + SGroupKeys key = {0}; + key.bytes = pCol->bytes; + key.type = pCol->type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pCol->bytes); + if (key.pData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush((*pGroupColVals), &key); + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + (*keyLen) += nullFlagSize; + + (*keyBuf) = taosMemoryCalloc(1, (*keyLen)); + if ((*keyBuf) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static void fillGroupKeyValsFromTagCols(SArray* pCols, SArray* pGroupKeys, SSDataBlock* pBlock, int32_t rowIndex) { + SColumnDataAgg* pColAgg = NULL; + + size_t numOfGroupCols = taosArrayGetSize(pCols); + + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*) taosArrayGet(pCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + + // valid range check. todo: return error code. + if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) { + continue; + } + + if (pBlock->pBlockAgg != NULL) { + pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + + SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + pkey->isNull = true; + } else { + pkey->isNull = false; + char* val = colDataGetData(pColInfoData, rowIndex); + if (pkey->type == TSDB_DATA_TYPE_JSON) { + if (tTagIsJson(val)) { + terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; + return; + } + int32_t dataLen = getJsonValueLen(val); + memcpy(pkey->pData, val, dataLen); + } else if (IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, varDataTLen(val)); + ASSERT(varDataTLen(val) <= pkey->bytes); + } else { + memcpy(pkey->pData, val, pkey->bytes); + } + } + } +} + +static int32_t combineGroupKeysIntoBuf(void* pKey, const SArray* pGroupKeys) { + size_t numOfGroupCols = taosArrayGetSize(pGroupKeys); + + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); + if (pkey->isNull) { + isNull[i] = 1; + continue; + } + + isNull[i] = 0; + if (pkey->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(pkey->pData); + memcpy(pStart, (pkey->pData), dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pkey->type)) { + varDataCopy(pStart, pkey->pData); + pStart += varDataTLen(pkey->pData); + ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); + } else { + memcpy(pStart, pkey->pData, pkey->bytes); + pStart += pkey->bytes; + } + } + + return (int32_t)(pStart - (char*)pKey); +} + SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); @@ -153,6 +305,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } + pInfo->pTagEqualConditions = pJoinNode->pTagEqualCondtions; + pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); + extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); + initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); + initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -179,8 +338,22 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } +static void freeGroupKeyData(void* param) { + SGroupKeys* pKey = (SGroupKeys*)param; + taosMemoryFree(pKey->pData); +} + void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; + + taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->rightTagCols); + + taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->leftTagCols); + nodesDestroyNode(pJoinOperator->pCondAfterMerge); pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); @@ -300,22 +473,121 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } +static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj** ppHashObj) { + int32_t buildTableCap = MIN(taosArrayGetSize(rightRowLocations), 4096); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); + for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { + SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); + fillGroupKeyValsFromTagCols(pInfo->rightTagCols, pInfo->rightTagKeys, rightRow->pDataBlock, rightRow->pos); + int32_t keyLen = combineGroupKeysIntoBuf(pInfo->rightTagKeyBuf, pInfo->rightTagKeys); + SArray** ppRows = tSimpleHashGet(buildTable, pInfo->rightTagKeyBuf, keyLen); + if (!ppRows) { + SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); + taosArrayPush(rows, rightRow); + tSimpleHashPut(buildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); + } else { + taosArrayPush(*ppRows, rightRow); + } + } + *ppHashObj = buildTable; + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, + const SArray* leftRowLocations, int32_t leftRowIdx, + int32_t rightRowIdx, SSHashObj* rightTableHash, bool* pReachThreshold) { + *pReachThreshold = false; + uint32_t limitRowNum = pOperator->resultInfo.threshold; + SJoinOperatorInfo* pJoinInfo = pOperator->info; + size_t leftNumJoin = taosArrayGetSize(leftRowLocations); + + int32_t i,j; + + for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { + SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); + fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); + int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); + SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); + if (!ppRightRows) { + continue; + } + SArray* pRightRows = *ppRightRows; + size_t rightRowsSize = taosArrayGetSize(pRightRows); + for (j = rightRowIdx; j < rightRowsSize; ++j) { + if (*nRows >= limitRowNum) { + *pReachThreshold = true; + break; + } + + SRowLocation* rightRow = taosArrayGet(pRightRows, j); + mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, + rightRow->pos); + ++*nRows; + } + if (*pReachThreshold) { + break; + } + } + + if (*pReachThreshold) { + pJoinInfo->rowCtx.rowRemains = true; + pJoinInfo->rowCtx.leftRowIdx = i; + pJoinInfo->rowCtx.rightRowIdx = j; + } + return TSDB_CODE_SUCCESS; +} + +static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + + tSimpleHashCleanup(pBuildTable); +} + +static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, + SArray* rightCreatedBlocks, SSHashObj* rightTableHash) { + for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); + blockDataDestroy(pBlock); + } + taosArrayDestroy(rightCreatedBlocks); + for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); + blockDataDestroy(pBlock); + } + mergeJoinDestoryBuildTable(rightTableHash); + + taosArrayDestroy(leftCreatedBlocks); + taosArrayDestroy(leftRowLocations); + + pJoinInfo->rowCtx.rowRemains = false; + pJoinInfo->rowCtx.leftRowLocations = NULL; + pJoinInfo->rowCtx.leftCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightCreatedBlocks = NULL; + pJoinInfo->rowCtx.buildTableTSRange = NULL; +} + static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, int32_t* nRows) { int32_t code = TSDB_CODE_SUCCESS; SJoinOperatorInfo* pJoinInfo = pOperator->info; SArray* leftRowLocations = NULL; SArray* leftCreatedBlocks = NULL; - SArray* rightRowLocations = NULL; SArray* rightCreatedBlocks = NULL; int32_t leftRowIdx = 0; int32_t rightRowIdx = 0; - int32_t i, j; - + SSHashObj* rightTableHash = NULL; + if (pJoinInfo->rowCtx.rowRemains) { leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; - rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; + rightTableHash = pJoinInfo->rowCtx.buildTableTSRange; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; rightRowIdx = pJoinInfo->rowCtx.rightRowIdx; @@ -323,85 +595,42 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t leftRowLocations = taosArrayInit(8, sizeof(SRowLocation)); leftCreatedBlocks = taosArrayInit(8, POINTER_BYTES); - rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); + SArray* rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); rightCreatedBlocks = taosArrayInit(8, POINTER_BYTES); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 0, pJoinInfo->leftCol.slotId, pJoinInfo->pLeft, pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); + mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); + taosArrayDestroy(rightRowLocations); } size_t leftNumJoin = taosArrayGetSize(leftRowLocations); - size_t rightNumJoin = taosArrayGetSize(rightRowLocations); - uint32_t maxRowNum = *nRows + (leftNumJoin - leftRowIdx - 1) * rightNumJoin + rightNumJoin - rightRowIdx; - uint32_t limitRowNum = maxRowNum; - if (maxRowNum > pOperator->resultInfo.threshold) { - limitRowNum = pOperator->resultInfo.threshold; - if (!pJoinInfo->rowCtx.rowRemains) { + code = blockDataEnsureCapacity(pRes, pOperator->resultInfo.threshold); + if (code != TSDB_CODE_SUCCESS) { + qError("%s can not ensure block capacity for join. left: %zu", GET_TASKID(pOperator->pTaskInfo), + leftNumJoin); + } + + bool reachThreshold = false; + + if (code == TSDB_CODE_SUCCESS) { + mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, + rightRowIdx, rightTableHash, &reachThreshold); + } + + if (!reachThreshold) { + mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, + rightTableHash); + + } else { pJoinInfo->rowCtx.rowRemains = true; pJoinInfo->rowCtx.ts = timestamp; pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; - pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - } - } - - code = blockDataEnsureCapacity(pRes, limitRowNum); - if (code != TSDB_CODE_SUCCESS) { - qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo), - leftNumJoin, rightNumJoin); - } - - - if (code == TSDB_CODE_SUCCESS) { - bool done = false; - for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { - for (j = rightRowIdx; j < rightNumJoin; ++j) { - if (*nRows >= limitRowNum) { - done = true; - break; - } - - SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); - SRowLocation* rightRow = taosArrayGet(rightRowLocations, j); - mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, - rightRow->pos); - ++*nRows; - } - if (done) { - break; - } - } - - if (maxRowNum > pOperator->resultInfo.threshold) { - pJoinInfo->rowCtx.leftRowIdx = i; - pJoinInfo->rowCtx.rightRowIdx = j; - } - } - - if (maxRowNum <= pOperator->resultInfo.threshold) { - for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(rightCreatedBlocks); - taosArrayDestroy(rightRowLocations); - for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(leftCreatedBlocks); - taosArrayDestroy(leftRowLocations); - - if (pJoinInfo->rowCtx.rowRemains) { - pJoinInfo->rowCtx.rowRemains = false; - pJoinInfo->rowCtx.leftRowLocations = NULL; - pJoinInfo->rowCtx.rightRowLocations = NULL; - pJoinInfo->rowCtx.leftCreatedBlocks = NULL; - pJoinInfo->rowCtx.rightCreatedBlocks = NULL; - } + pJoinInfo->rowCtx.buildTableTSRange = rightTableHash; } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index f5ef02e207..8c111133fc 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -759,6 +759,14 @@ static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (OP_TYPE_EQUAL != pOper->opType) { return false; } + if (QUERY_NODE_COLUMN != nodeType(pOper->pLeft) || QUERY_NODE_COLUMN != nodeType(pOper->pRight)) { + return false; + } + SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft); + SColumnNode* pRight = (SColumnNode*)(pOper->pRight); + if (pLeft->node.resType.type != pRight->node.resType.type) { + return false; + } SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; if (pushDownCondOptIsTag(pOper->pLeft, pLeftCols)) { From a51de5cc9384b22d09ec27f9607f46361e548975 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 16 May 2023 16:31:58 +0800 Subject: [PATCH 009/187] fix: add restriction --- source/libs/planner/src/planOptimizer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8c111133fc..d76c25fc55 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -764,7 +764,8 @@ static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { } SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft); SColumnNode* pRight = (SColumnNode*)(pOper->pRight); - if (pLeft->node.resType.type != pRight->node.resType.type) { + //TODO: add cast to operator and remove this restriction of optimization + if (pLeft->node.resType.type != pRight->node.resType.type || pLeft->node.resType.bytes != pRight->node.resType.bytes) { return false; } SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; From 568725a245d8e1a2f80dc056c85955bdd8a382cd Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 17 May 2023 08:33:47 +0800 Subject: [PATCH 010/187] fix: using right rows to build hash table when the tag equal cond is not null and same ts row row number is greater than 16 --- source/libs/executor/src/joinoperator.c | 74 ++++++++++++++++--------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index a5591b22b3..54ddab9e1e 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -34,7 +34,9 @@ typedef struct SJoinRowCtx { SArray* rightCreatedBlocks; int32_t leftRowIdx; int32_t rightRowIdx; + SSHashObj* buildTableTSRange; + SArray* rightRowLocations; } SJoinRowCtx; typedef struct SJoinOperatorInfo { @@ -306,12 +308,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t } pInfo->pTagEqualConditions = pJoinNode->pTagEqualCondtions; - pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); - pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); - extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); - initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); - initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); - + if (pInfo->pTagEqualConditions != NULL) { + pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); + extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); + initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); + initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -345,15 +348,15 @@ static void freeGroupKeyData(void* param) { void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; + if (pJoinOperator->pTagEqualConditions != NULL) { + taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->rightTagCols); - taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); - taosArrayDestroy(pJoinOperator->rightTagCols); - - taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); - taosArrayDestroy(pJoinOperator->leftTagCols); - + taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->leftTagCols); + } nodesDestroyNode(pJoinOperator->pCondAfterMerge); pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); @@ -496,7 +499,7 @@ static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* right static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, const SArray* leftRowLocations, int32_t leftRowIdx, - int32_t rightRowIdx, SSHashObj* rightTableHash, bool* pReachThreshold) { + int32_t rightRowIdx, SSHashObj* rightTableHash, SArray* rightRowLocations, bool* pReachThreshold) { *pReachThreshold = false; uint32_t limitRowNum = pOperator->resultInfo.threshold; SJoinOperatorInfo* pJoinInfo = pOperator->info; @@ -506,13 +509,18 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); - fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); - int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); - SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); - if (!ppRightRows) { - continue; + SArray* pRightRows = NULL; + if (rightTableHash != NULL) { + fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); + int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); + SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); + if (!ppRightRows) { + continue; + } + pRightRows = *ppRightRows; + } else { + pRightRows = rightRowLocations; } - SArray* pRightRows = *ppRightRows; size_t rightRowsSize = taosArrayGetSize(pRightRows); for (j = rightRowIdx; j < rightRowsSize; ++j) { if (*nRows >= limitRowNum) { @@ -551,7 +559,7 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { } static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, - SArray* rightCreatedBlocks, SSHashObj* rightTableHash) { + SArray* rightCreatedBlocks, SSHashObj* rightTableHash, SArray* rightRowLocations) { for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); blockDataDestroy(pBlock); @@ -561,7 +569,12 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); blockDataDestroy(pBlock); } - mergeJoinDestoryBuildTable(rightTableHash); + if (rightRowLocations != NULL) { + taosArrayDestroy(rightRowLocations); + } + if (rightTableHash != NULL) { + mergeJoinDestoryBuildTable(rightTableHash); + } taosArrayDestroy(leftCreatedBlocks); taosArrayDestroy(leftRowLocations); @@ -571,6 +584,7 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef pJoinInfo->rowCtx.leftCreatedBlocks = NULL; pJoinInfo->rowCtx.rightCreatedBlocks = NULL; pJoinInfo->rowCtx.buildTableTSRange = NULL; + pJoinInfo->rowCtx.rightRowLocations = NULL; } static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, @@ -578,6 +592,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t int32_t code = TSDB_CODE_SUCCESS; SJoinOperatorInfo* pJoinInfo = pOperator->info; SArray* leftRowLocations = NULL; + SArray* rightRowLocations = NULL; SArray* leftCreatedBlocks = NULL; SArray* rightCreatedBlocks = NULL; int32_t leftRowIdx = 0; @@ -588,6 +603,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; rightTableHash = pJoinInfo->rowCtx.buildTableTSRange; + rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; rightRowIdx = pJoinInfo->rowCtx.rightRowIdx; @@ -602,8 +618,11 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); - mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); - taosArrayDestroy(rightRowLocations); + if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); + taosArrayDestroy(rightRowLocations); + rightRowLocations = NULL; + } } size_t leftNumJoin = taosArrayGetSize(leftRowLocations); @@ -617,12 +636,12 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t if (code == TSDB_CODE_SUCCESS) { mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, - rightRowIdx, rightTableHash, &reachThreshold); + rightRowIdx, rightTableHash, rightRowLocations, &reachThreshold); } if (!reachThreshold) { mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, - rightTableHash); + rightTableHash, rightRowLocations); } else { pJoinInfo->rowCtx.rowRemains = true; @@ -631,6 +650,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; pJoinInfo->rowCtx.buildTableTSRange = rightTableHash; + pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; } From 03e77288e8ca2b3ea9a7385c21f181593fa899a6 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 17 May 2023 10:07:19 +0800 Subject: [PATCH 011/187] fix: add on conditions null check and rightRowLocations no longer hides the outer scope rightRowLocations --- source/libs/executor/src/joinoperator.c | 4 ++-- source/libs/planner/src/planOptimizer.c | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 54ddab9e1e..07b8a4db11 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -477,7 +477,7 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator } static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj** ppHashObj) { - int32_t buildTableCap = MIN(taosArrayGetSize(rightRowLocations), 4096); + int32_t buildTableCap = TMIN(taosArrayGetSize(rightRowLocations), 256); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { @@ -611,7 +611,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t leftRowLocations = taosArrayInit(8, sizeof(SRowLocation)); leftCreatedBlocks = taosArrayInit(8, POINTER_BYTES); - SArray* rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); + rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); rightCreatedBlocks = taosArrayInit(8, POINTER_BYTES); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 0, pJoinInfo->leftCol.slotId, pJoinInfo->pLeft, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d76c25fc55..5be67389c8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -806,6 +806,10 @@ static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin } static int32_t pushDownCondOptJoinExtractTagEqualCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (NULL == pJoin->pOnConditions) { + pJoin->pTagEqualConditions = NULL; + return TSDB_CODE_SUCCESS; + } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { return pushDownCondOptJoinExtractTagEqualLogicCond(pJoin); From 28b098c824a7e008f6317d85b91ea29d15159d0a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 May 2023 10:47:11 +0800 Subject: [PATCH 012/187] fix: union clause error message --- source/util/src/terror.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6f726c7eff..712c413e7c 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -372,7 +372,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many groups/time window in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") From 1ded7a052bfc5fd9e823236d35800c5dc1fd3996 Mon Sep 17 00:00:00 2001 From: Yu Sun Date: Wed, 17 May 2023 07:22:15 +0000 Subject: [PATCH 013/187] docs(connector-rust): update documents of connector for rust The connector for rust has been updated to use `deadpool` as default connection pool instead of `r2d2`, and the `r2d2` was only kept for blocking mode. --- docs/zh/08-connector/26-rust.mdx | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index eeb93a6558..5cf37f7946 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -64,6 +64,13 @@ taos = "*" taos = { version = "*", default-features = false, features = ["ws"] } ``` +当仅启用 `ws` 特性时，可同时指定 `r2d2` 使得在同步（blocking/sync）模式下使用 [r2d2] 作为连接池： + +```toml +[dependencies] +taos = { version = "*", default-features = false, features = ["r2d2", "ws"] } +``` + @@ -252,26 +259,24 @@ let conn: Taos = cfg.build(); ### 连接池 -在复杂应用中，建议启用连接池。[taos] 的连接池使用 [r2d2] 实现。 +在复杂应用中，建议启用连接池。[taos] 的连接池默认（异步模式）使用 [deadpool] 实现。 如下，可以生成一个默认参数的连接池。 ```rust -let pool = TaosBuilder::from_dsn(dsn)?.pool()?; +let pool: Pool = TaosBuilder::from_dsn("taos:///") + .unwrap() + .pool() + .unwrap(); ``` 同样可以使用连接池的构造器，对连接池参数进行设置： ```rust -let dsn = "taos://localhost:6030"; - -let opts = PoolBuilder::new() - .max_size(5000) // max connections - .max_lifetime(Some(Duration::from_secs(60 * 60))) // lifetime of each connection - .min_idle(Some(1000)) // minimal idle connections - .connection_timeout(Duration::from_secs(2)); - -let pool = TaosBuilder::from_dsn(dsn)?.with_pool_builder(opts)?; +let pool: Pool = Pool::builder(Manager::from_dsn(self.dsn.clone()).unwrap().0) + .max_size(88) // 最大连接数 + .build() + .unwrap(); ``` 在应用代码中，使用 `pool.get()?` 来获取一个连接对象 [Taos]。 @@ -511,6 +516,7 @@ consumer.unsubscribe().await; 其他相关结构体 API 使用说明请移步 Rust 文档托管网页：。 [taos]: https://github.com/taosdata/rust-connector-taos +[deadpool]: https://crates.io/crates/deadpool [r2d2]: https://crates.io/crates/r2d2 [TaosBuilder]: https://docs.rs/taos/latest/taos/struct.TaosBuilder.html [TaosCfg]: https://docs.rs/taos/latest/taos/struct.TaosCfg.html From ab7cb995a2eb87fd9c32168d1874cf99a3f07012 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 May 2023 15:31:53 +0800 Subject: [PATCH 014/187] fix: join push down condition issue --- source/libs/nodes/src/nodesEqualFuncs.c | 4 +--- source/libs/parser/test/parSelectTest.cpp | 2 ++ source/libs/planner/src/planPhysiCreater.c | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 156744ef1d..477d27d271 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -82,9 +82,7 @@ static bool columnNodeEqual(const SColumnNode* a, const SColumnNode* b) { COMPARE_STRING_FIELD(dbName); COMPARE_STRING_FIELD(tableName); COMPARE_STRING_FIELD(colName); - if (0 == a->tableId) { - COMPARE_STRING_FIELD(tableAlias); - } + COMPARE_STRING_FIELD(tableAlias); return true; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index ec6c69ea8d..c60257cc22 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -459,6 +459,8 @@ TEST_F(ParserSelectTest, joinSemanticCheck) { run("SELECT * FROM (SELECT tag1, SUM(c1) s FROM st1 GROUP BY tag1) t1, st1 t2 where t1.tag1 = t2.tag1", TSDB_CODE_PAR_NOT_SUPPORT_JOIN); + + run("SELECT count(*) FROM t1 a join t1 b on a.ts=b.ts where a.ts=b.ts"); } } // namespace ParserTest diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e2c2e4c655..651be08aec 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1643,6 +1643,9 @@ static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pChildren, (SNode*)pChild); } + if (TSDB_CODE_SUCCESS != code) { + break; + } } if (TSDB_CODE_SUCCESS == code) { From 73b3b3fc377be4480112e4f61f32f1959e79ac24 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 May 2023 15:48:12 +0800 Subject: [PATCH 015/187] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6724e2d303..eb4f04c914 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -291,13 +291,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - if (offset.val.type == TMQ_OFFSET__LOG) { - STqHandle* pHandle = taosHashGet(pTq->pHandle, offset.subKey, strlen(offset.subKey)); - if (pHandle && (walRefVer(pHandle->pRef, offset.val.version) < 0)) { - return -1; - } - } - return 0; } From 4bb25c280f0452c5b078580c1bba8b9b09a94b91 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 15:54:36 +0800 Subject: [PATCH 016/187] fix:add log --- source/dnode/vnode/src/tq/tq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9730ad2877..ff715f1828 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -359,7 +359,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } while (tqIsHandleExec(pHandle)) { - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", consumerId, vgId, req.subKey); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosMsleep(5); } @@ -372,6 +372,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } tqSetHandleExec(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); // 3. update the epoch value @@ -389,6 +390,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); tqSetHandleIdle(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; } @@ -555,7 +557,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg goto end; } else { while (tqIsHandleExec(pHandle)) { - tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); + tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); taosMsleep(5); } From 1d9dd153eacdb10e5e1d03d896ca104f1bcaf30b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 18:57:45 +0800 Subject: [PATCH 017/187] fix:error in pHandle lock --- source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/tq/tq.c | 64 +++++++++++++++------------- source/dnode/vnode/src/tq/tqMeta.c | 2 - source/dnode/vnode/src/tq/tqOffset.c | 2 - source/dnode/vnode/src/tq/tqRead.c | 2 - 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ef36b8429a..f0008f04cb 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -93,6 +93,7 @@ typedef struct { typedef enum tq_handle_status{ TMQ_HANDLE_STATUS_IDLE = 0, TMQ_HANDLE_STATUS_EXEC = 1, + TMQ_HANDLE_STATUS_DELETE = 2, }tq_handle_status; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ff715f1828..b1fca3faa8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -25,6 +25,7 @@ static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_EXEC;} static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_IDLE;} +static FORCE_INLINE void tqSetHandleDelete(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_DELETE;} int32_t tqInit() { int8_t old; @@ -297,13 +298,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } if (offset.val.type == TMQ_OFFSET__LOG) { - taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, offset.subKey, strlen(offset.subKey)); if (pHandle && (walSetRefVer(pHandle->pRef, offset.val.version) < 0)) { - taosWUnLockLatch(&pTq->lock); return -1; } - taosWUnLockLatch(&pTq->lock); } return 0; @@ -337,6 +335,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SMqPollReq req = {0}; + int code = 0; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); terrno = TSDB_CODE_INVALID_MSG; @@ -347,20 +346,29 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t reqEpoch = req.epoch; STqOffsetVal reqOffset = req.reqOffset; int32_t vgId = TD_VID(pTq->pVnode); + STqHandle* pHandle = NULL; - taosWLockLatch(&pTq->lock); - // 1. find handle - STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - if (pHandle == NULL) { - tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); - terrno = TSDB_CODE_INVALID_MSG; + while (1) { + taosWLockLatch(&pTq->lock); + // 1. find handle + pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle == NULL) { + tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; + taosWUnLockLatch(&pTq->lock); + return -1; + } + + bool exec = tqIsHandleExec(pHandle); + if(!exec) { + tqSetHandleExec(pHandle); + taosWUnLockLatch(&pTq->lock); + break; + } taosWUnLockLatch(&pTq->lock); - return -1; - } - while (tqIsHandleExec(pHandle)) { tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); - taosMsleep(5); + taosMsleep(10); } // 2. check re-balance status @@ -368,12 +376,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - taosWUnLockLatch(&pTq->lock); - return -1; + code = -1; + goto end; } - tqSetHandleExec(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); - taosWUnLockLatch(&pTq->lock); // 3. update the epoch value int32_t savedEpoch = pHandle->epoch; @@ -388,7 +395,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); - int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); + code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); + +end: tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; @@ -405,8 +414,8 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { while (tqIsHandleExec(pHandle)) { - tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); - taosMsleep(5); + tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); + taosMsleep(10); } if (pHandle->pRef) { walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); @@ -473,7 +482,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); - taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -556,20 +564,14 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { - while (tqIsHandleExec(pHandle)) { - tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); - taosMsleep(5); - } - if (pHandle->consumerId == req.newConsumerId) { // do nothing tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); - atomic_add_fetch_32(&pHandle->epoch, 1); - +// atomic_add_fetch_32(&pHandle->epoch, 1); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); - atomic_store_32(&pHandle->epoch, 0); +// atomic_store_32(&pHandle->epoch, 0); } // kill executing task qTaskInfo_t pTaskInfo = pHandle->execHandle.task; @@ -580,13 +582,15 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg qStreamCloseTsdbReader(pTaskInfo); } // remove if it has been register in the push manager, and return one empty block to consumer + taosWLockLatch(&pTq->lock); tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } end: - taosWUnLockLatch(&pTq->lock); taosMemoryFree(req.qmsg); return ret; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 5654147b6d..b3d21887c9 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -352,9 +352,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); - taosWLockLatch(&pTq->lock); taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); - taosWUnLockLatch(&pTq->lock); } end: diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 377a5d1887..0a9905b544 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -78,7 +78,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // todo remove this if (offset.val.type == TMQ_OFFSET__LOG) { - taosWLockLatch(&pStore->pTq->lock); STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); if (pHandle) { if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) { @@ -86,7 +85,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // offset.val.version); } } - taosWUnLockLatch(&pStore->pTq->lock); } taosMemoryFree(pMemBuf); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 09574a6b62..a26a749af3 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1035,7 +1035,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t vgId = TD_VID(pTq->pVnode); // update the table list for each consumer handle - taosWLockLatch(&pTq->lock); while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); if (pIter == NULL) { @@ -1092,7 +1091,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } } - taosWUnLockLatch(&pTq->lock); // update the table list handle for each stream scanner/wal reader taosWLockLatch(&pTq->pStreamMeta->lock); From 405cdfa7bc5637be46072149303bed2a03e5b20c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 19:02:16 +0800 Subject: [PATCH 018/187] fix:error in pHandle lock --- source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/tq/tq.c | 1 - 2 files changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f0008f04cb..ef36b8429a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -93,7 +93,6 @@ typedef struct { typedef enum tq_handle_status{ TMQ_HANDLE_STATUS_IDLE = 0, TMQ_HANDLE_STATUS_EXEC = 1, - TMQ_HANDLE_STATUS_DELETE = 2, }tq_handle_status; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b1fca3faa8..d8ebf72549 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -25,7 +25,6 @@ static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_EXEC;} static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_IDLE;} -static FORCE_INLINE void tqSetHandleDelete(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_DELETE;} int32_t tqInit() { int8_t old; From cf2c5f190c865003cc7c8132a158840c1e07d8b9 Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Wed, 17 May 2023 19:26:34 +0800 Subject: [PATCH 019/187] Update cases.task modify non asan mode --- tests/parallel_test/cases.task | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 5a6ec7a825..8d151f4756 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -78,7 +78,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +,,n,system-test,python3 ./test.py -f 7-tmq/tmqConsumerGroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAlterSchema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 From 4d5bd2b6dee5321d3aa4695c5553f1866cf4a37e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 May 2023 19:40:21 +0800 Subject: [PATCH 020/187] fix(stream): fix memory leak for stream processing. --- source/libs/stream/src/stream.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 7047461ca8..f4ee0b98b7 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -297,6 +297,7 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); streamDataSubmitDestroy(pSubmitBlock); + taosFreeQitem(pSubmitBlock); return -1; } From 54f48601394cf1dff9f8933d57a3887aaa8ec0d4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 19:44:56 +0800 Subject: [PATCH 021/187] fix:rollback [TD-23972] push subscribe msg to vnode even though consumer not change --- source/dnode/mnode/impl/src/mndSubscribe.c | 21 +++------------ source/dnode/vnode/src/tq/tq.c | 30 ++++++++++------------ 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e62102fa77..2ae4ede25a 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -133,10 +133,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg) { -// if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { -// terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; -// return -1; -// } + if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { + terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; + return -1; + } void *buf; int32_t tlen; @@ -269,18 +269,6 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { } } -static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ - for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ - SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); - SMqRebOutputVg outputVg = { - .oldConsumerId = pConsumerEp->consumerId, - .newConsumerId = pConsumerEp->consumerId, - .pVgEp = pVgEp, - }; - taosArrayPush(pOutput->rebVgs, &outputVg); - } -} - static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, int32_t imbConsumerNum) { const char *pSubKey = pOutput->pSub->key; @@ -330,7 +318,6 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas } } } - putNoTransferToOutput(pOutput, pConsumerEp); } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d8ebf72549..e074a2d8a6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -564,29 +564,27 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg goto end; } else { if (pHandle->consumerId == req.newConsumerId) { // do nothing - tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); + tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs, should not reach here", req.vgId, req.newConsumerId); // atomic_add_fetch_32(&pHandle->epoch, 1); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); // atomic_store_32(&pHandle->epoch, 0); + // kill executing task + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + } + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } + // remove if it has been register in the push manager, and return one empty block to consumer + taosWLockLatch(&pTq->lock); + tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } - // remove if it has been register in the push manager, and return one empty block to consumer - taosWLockLatch(&pTq->lock); - tqUnregisterPushHandle(pTq, pHandle); - taosWUnLockLatch(&pTq->lock); - - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); - goto end; } end: From 8994081592b788630e784fa3d7894ba9b9cffeff Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 May 2023 10:25:10 +0800 Subject: [PATCH 022/187] fix: memory size overflow issue --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fa46715c22..5953074825 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -135,7 +135,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in size_t keyLen = 0; int32_t iter = 0; - int32_t bufLen = 0, offset = 0; + int64_t bufLen = 0, offset = 0; // todo move away and record this during create window while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { From ee16b46112e6a83eb1434b9ab90cc104c8e89bcb Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 10:46:19 +0800 Subject: [PATCH 023/187] fix: remove group keys from join operator --- source/libs/executor/src/joinoperator.c | 94 +++++-------------------- 1 file changed, 18 insertions(+), 76 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 07b8a4db11..68f6951c26 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -55,12 +55,10 @@ typedef struct SJoinOperatorInfo { SNode* pTagEqualConditions; SArray* leftTagCols; - SArray* leftTagKeys; char* leftTagKeyBuf; int32_t leftTagKeyLen; SArray* rightTagCols; - SArray* rightTagKeys; char* rightTagKeyBuf; int32_t rightTagKeyLen; @@ -141,27 +139,11 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD } } -static int32_t initTagColsGroupkeys(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { - *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); - if ((*pGroupColVals) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - +static int32_t initTagColskeyBuf(int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); (*keyLen) += pCol->bytes; // actual data + null_flag - - SGroupKeys key = {0}; - key.bytes = pCol->bytes; - key.type = pCol->type; - key.isNull = false; - key.pData = taosMemoryCalloc(1, pCol->bytes); - if (key.pData == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - taosArrayPush((*pGroupColVals), &key); } int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; @@ -175,10 +157,11 @@ static int32_t initTagColsGroupkeys(SArray** pGroupColVals, int32_t* keyLen, cha return TSDB_CODE_SUCCESS; } -static void fillGroupKeyValsFromTagCols(SArray* pCols, SArray* pGroupKeys, SSDataBlock* pBlock, int32_t rowIndex) { +static int32_t fillKeyBufFromTagCols(SArray* pCols, SSDataBlock* pBlock, int32_t rowIndex, void* pKey) { SColumnDataAgg* pColAgg = NULL; - size_t numOfGroupCols = taosArrayGetSize(pCols); + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = (SColumn*) taosArrayGet(pCols, i); @@ -193,56 +176,24 @@ static void fillGroupKeyValsFromTagCols(SArray* pCols, SArray* pGroupKeys, SSDat pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } - SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { - pkey->isNull = true; + isNull[i] = 1; } else { - pkey->isNull = false; + isNull[i] = 0; char* val = colDataGetData(pColInfoData, rowIndex); - if (pkey->type == TSDB_DATA_TYPE_JSON) { - if (tTagIsJson(val)) { - terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; - return; - } + if (pCol->type == TSDB_DATA_TYPE_JSON) { int32_t dataLen = getJsonValueLen(val); - memcpy(pkey->pData, val, dataLen); - } else if (IS_VAR_DATA_TYPE(pkey->type)) { - memcpy(pkey->pData, val, varDataTLen(val)); - ASSERT(varDataTLen(val) <= pkey->bytes); + memcpy(pStart, val, dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + varDataCopy(pStart, val); + pStart += varDataTLen(val); } else { - memcpy(pkey->pData, val, pkey->bytes); + memcpy(pStart, val, pCol->bytes); + pStart += pCol->bytes; } } } -} - -static int32_t combineGroupKeysIntoBuf(void* pKey, const SArray* pGroupKeys) { - size_t numOfGroupCols = taosArrayGetSize(pGroupKeys); - - char* isNull = (char*)pKey; - char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; - for (int32_t i = 0; i < numOfGroupCols; ++i) { - SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); - if (pkey->isNull) { - isNull[i] = 1; - continue; - } - - isNull[i] = 0; - if (pkey->type == TSDB_DATA_TYPE_JSON) { - int32_t dataLen = getJsonValueLen(pkey->pData); - memcpy(pStart, (pkey->pData), dataLen); - pStart += dataLen; - } else if (IS_VAR_DATA_TYPE(pkey->type)) { - varDataCopy(pStart, pkey->pData); - pStart += varDataTLen(pkey->pData); - ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); - } else { - memcpy(pStart, pkey->pData, pkey->bytes); - pStart += pkey->bytes; - } - } - return (int32_t)(pStart - (char*)pKey); } @@ -312,8 +263,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); - initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); - initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + initTagColskeyBuf(&pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); + initTagColskeyBuf(&pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); @@ -341,20 +292,13 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } -static void freeGroupKeyData(void* param) { - SGroupKeys* pKey = (SGroupKeys*)param; - taosMemoryFree(pKey->pData); -} - void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; if (pJoinOperator->pTagEqualConditions != NULL) { taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); taosArrayDestroy(pJoinOperator->rightTagCols); taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); taosArrayDestroy(pJoinOperator->leftTagCols); } nodesDestroyNode(pJoinOperator->pCondAfterMerge); @@ -482,8 +426,7 @@ static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* right SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); - fillGroupKeyValsFromTagCols(pInfo->rightTagCols, pInfo->rightTagKeys, rightRow->pDataBlock, rightRow->pos); - int32_t keyLen = combineGroupKeysIntoBuf(pInfo->rightTagKeyBuf, pInfo->rightTagKeys); + int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); SArray** ppRows = tSimpleHashGet(buildTable, pInfo->rightTagKeyBuf, keyLen); if (!ppRows) { SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); @@ -511,8 +454,7 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SArray* pRightRows = NULL; if (rightTableHash != NULL) { - fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); - int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); + int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftTagCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftTagKeyBuf); SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); if (!ppRightRows) { continue; From 14d4a81e750303da755b121da8c2c36dfac93dff Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 11:15:04 +0800 Subject: [PATCH 024/187] fix: build hash table is created during creating join operator --- source/libs/executor/src/joinoperator.c | 62 +++++++++++++------------ 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 68f6951c26..c90c18e6f5 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -35,7 +35,7 @@ typedef struct SJoinRowCtx { int32_t leftRowIdx; int32_t rightRowIdx; - SSHashObj* buildTableTSRange; + bool rightUseBuildTable; SArray* rightRowLocations; } SJoinRowCtx; @@ -62,6 +62,7 @@ typedef struct SJoinOperatorInfo { char* rightTagKeyBuf; int32_t rightTagKeyLen; + SSHashObj* rightBuildTable; SJoinRowCtx rowCtx; } SJoinOperatorInfo; @@ -265,6 +266,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); initTagColskeyBuf(&pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); initTagColskeyBuf(&pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->rightBuildTable = tSimpleHashInit(256, hashFn); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); @@ -292,9 +295,22 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } +static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + + tSimpleHashCleanup(pBuildTable); +} + void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; if (pJoinOperator->pTagEqualConditions != NULL) { + mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); taosArrayDestroy(pJoinOperator->rightTagCols); @@ -420,10 +436,7 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } -static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj** ppHashObj) { - int32_t buildTableCap = TMIN(taosArrayGetSize(rightRowLocations), 256); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); +static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj* buildTable) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); @@ -436,13 +449,12 @@ static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* right taosArrayPush(*ppRows, rightRow); } } - *ppHashObj = buildTable; return TSDB_CODE_SUCCESS; } static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, const SArray* leftRowLocations, int32_t leftRowIdx, - int32_t rightRowIdx, SSHashObj* rightTableHash, SArray* rightRowLocations, bool* pReachThreshold) { + int32_t rightRowIdx, bool useBuildTableTSRange, SArray* rightRowLocations, bool* pReachThreshold) { *pReachThreshold = false; uint32_t limitRowNum = pOperator->resultInfo.threshold; SJoinOperatorInfo* pJoinInfo = pOperator->info; @@ -453,9 +465,9 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SArray* pRightRows = NULL; - if (rightTableHash != NULL) { + if (useBuildTableTSRange) { int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftTagCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftTagKeyBuf); - SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); + SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftTagKeyBuf, keyLen); if (!ppRightRows) { continue; } @@ -488,20 +500,8 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* return TSDB_CODE_SUCCESS; } -static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { - void* p = NULL; - int32_t iter = 0; - - while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { - SArray* rows = (*(SArray**)p); - taosArrayDestroy(rows); - } - - tSimpleHashCleanup(pBuildTable); -} - static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, - SArray* rightCreatedBlocks, SSHashObj* rightTableHash, SArray* rightRowLocations) { + SArray* rightCreatedBlocks, bool rightUseBuildTable, SArray* rightRowLocations) { for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); blockDataDestroy(pBlock); @@ -514,8 +514,8 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef if (rightRowLocations != NULL) { taosArrayDestroy(rightRowLocations); } - if (rightTableHash != NULL) { - mergeJoinDestoryBuildTable(rightTableHash); + if (rightUseBuildTable) { + tSimpleHashClear(pJoinInfo->rightBuildTable); } taosArrayDestroy(leftCreatedBlocks); @@ -525,7 +525,7 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef pJoinInfo->rowCtx.leftRowLocations = NULL; pJoinInfo->rowCtx.leftCreatedBlocks = NULL; pJoinInfo->rowCtx.rightCreatedBlocks = NULL; - pJoinInfo->rowCtx.buildTableTSRange = NULL; + pJoinInfo->rowCtx.rightUseBuildTable = false; pJoinInfo->rowCtx.rightRowLocations = NULL; } @@ -540,11 +540,12 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t int32_t leftRowIdx = 0; int32_t rightRowIdx = 0; SSHashObj* rightTableHash = NULL; + bool rightUseBuildTable = false; if (pJoinInfo->rowCtx.rowRemains) { leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; - rightTableHash = pJoinInfo->rowCtx.buildTableTSRange; + rightUseBuildTable = pJoinInfo->rowCtx.rightUseBuildTable; rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; @@ -561,7 +562,8 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { - mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); + mergeJoinFillBuildTable(pJoinInfo, rightRowLocations, pJoinInfo->rightBuildTable); + rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); rightRowLocations = NULL; } @@ -578,12 +580,12 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t if (code == TSDB_CODE_SUCCESS) { mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, - rightRowIdx, rightTableHash, rightRowLocations, &reachThreshold); + rightRowIdx, rightUseBuildTable, rightRowLocations, &reachThreshold); } if (!reachThreshold) { mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, - rightTableHash, rightRowLocations); + rightUseBuildTable, rightRowLocations); } else { pJoinInfo->rowCtx.rowRemains = true; @@ -591,7 +593,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - pJoinInfo->rowCtx.buildTableTSRange = rightTableHash; + pJoinInfo->rowCtx.rightUseBuildTable = true; pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; From d7f8757844da2a8db1d9a756d2d9884a3b57cb91 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 11:23:50 +0800 Subject: [PATCH 025/187] fix: memory leak --- source/libs/executor/src/joinoperator.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index c90c18e6f5..0add034673 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -515,6 +515,12 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef taosArrayDestroy(rightRowLocations); } if (rightUseBuildTable) { + void* p = NULL; + int32_t iter = 0; + while ((p = tSimpleHashIterate(pJoinInfo->rightBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } tSimpleHashClear(pJoinInfo->rightBuildTable); } From 7102c107ebd4884541bfa47d428b73020b5a7d43 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 13:19:52 +0800 Subject: [PATCH 026/187] fix: remove parameter of filling build table --- source/libs/executor/src/joinoperator.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 0add034673..23bd7c5700 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -436,15 +436,16 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } -static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj* buildTable) { +static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { + tSimpleHashClear(pInfo->rightBuildTable); SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); - SArray** ppRows = tSimpleHashGet(buildTable, pInfo->rightTagKeyBuf, keyLen); + SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen); if (!ppRows) { SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); taosArrayPush(rows, rightRow); - tSimpleHashPut(buildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); + tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); } else { taosArrayPush(*ppRows, rightRow); } @@ -568,7 +569,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { - mergeJoinFillBuildTable(pJoinInfo, rightRowLocations, pJoinInfo->rightBuildTable); + mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); rightRowLocations = NULL; From 51cb79fc1d93eec01fba74a513c7e7993c3c8e98 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 13:28:46 +0800 Subject: [PATCH 027/187] fix: set rightUseBuildTable correctly --- source/libs/executor/src/joinoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 23bd7c5700..700a59ccd8 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -600,7 +600,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - pJoinInfo->rowCtx.rightUseBuildTable = true; + pJoinInfo->rowCtx.rightUseBuildTable = rightUseBuildTable; pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; From 72e63195920db702e3b1165949d1f13e56d84c53 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 May 2023 14:41:32 +0800 Subject: [PATCH 028/187] enh: optimize order by for partition by clause --- source/libs/planner/src/planLogicCreater.c | 23 ++++--- tests/script/tsim/query/partitionby.sim | 79 ++++++++++++++++++++-- 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b9ea5ba0cd..91a5e6ac93 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -37,19 +37,24 @@ typedef struct SRewriteExprCxt { int32_t errCode; SNodeList* pExprs; bool* pOutputs; + bool isPartitionBy; } SRewriteExprCxt; -static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { +static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartitionBy) { switch (pFunc->funcType) { case FUNCTION_TYPE_TBNAME: pCol->colType = COLUMN_TYPE_TBNAME; break; case FUNCTION_TYPE_WSTART: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_START; break; case FUNCTION_TYPE_WEND: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_END; break; case FUNCTION_TYPE_WDURATION: @@ -103,7 +108,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { - setColumnInfo((SFunctionNode*)pExpr, pCol); + setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; @@ -142,7 +147,8 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExpr(pExpr, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); if (TSDB_CODE_SUCCESS == cxt.errCode) { nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); @@ -170,7 +176,8 @@ static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList** static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause, SNodeList** pRewriteExprs) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; if (NULL != pRewriteExprs) { cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool)); if (NULL == cxt.pOutputs) { @@ -187,14 +194,14 @@ static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ES static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } diff --git a/tests/script/tsim/query/partitionby.sim b/tests/script/tsim/query/partitionby.sim index 8babd1aa8d..4657713780 100644 --- a/tests/script/tsim/query/partitionby.sim +++ b/tests/script/tsim/query/partitionby.sim @@ -24,16 +24,83 @@ sql use $db sql create table $mt1 (ts timestamp, f1 int) TAGS(tag1 int, tag2 binary(500)) sql create table tb0 using $mt1 tags(0, 'a'); sql create table tb1 using $mt1 tags(1, 'b'); -sql create table tb2 using $mt1 tags(1, 'a'); -sql create table tb3 using $mt1 tags(1, 'a'); -sql create table tb4 using $mt1 tags(3, 'b'); -sql create table tb5 using $mt1 tags(3, 'a'); -sql create table tb6 using $mt1 tags(3, 'b'); -sql create table tb7 using $mt1 tags(3, 'b'); +sql create table tb2 using $mt1 tags(2, 'a'); +sql create table tb3 using $mt1 tags(3, 'a'); +sql create table tb4 using $mt1 tags(4, 'b'); +sql create table tb5 using $mt1 tags(5, 'a'); +sql create table tb6 using $mt1 tags(6, 'b'); +sql create table tb7 using $mt1 tags(7, 'b'); sql select * from $mt1 partition by tag1,tag2 limit 1; if $rows != 0 then return -1 endi +sql insert into tb0 values ('2022-04-26 15:15:08', 1); +sql insert into tb1 values ('2022-04-26 15:15:07', 2); +sql insert into tb2 values ('2022-04-26 15:15:06', 3); +sql insert into tb3 values ('2022-04-26 15:15:05', 4); +sql insert into tb4 values ('2022-04-26 15:15:04', 5); +sql insert into tb5 values ('2022-04-26 15:15:03', 6); +sql insert into tb6 values ('2022-04-26 15:15:02', 7); +sql insert into tb7 values ('2022-04-26 15:15:01', 8); + +sql select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart; +if $rows != 8 then + return -1 +endi +if $data00 != @22-04-26 15:15:01.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data10 != @22-04-26 15:15:02.000@ then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data20 != @22-04-26 15:15:03.000@ then + return -1 +endi +if $data21 != 1 then + return -1 +endi +if $data30 != @22-04-26 15:15:04.000@ then + return -1 +endi +if $data31 != 1 then + return -1 +endi +if $data40 != @22-04-26 15:15:05.000@ then + return -1 +endi +if $data41 != 1 then + return -1 +endi +if $data50 != @22-04-26 15:15:06.000@ then + return -1 +endi +if $data51 != 1 then + return -1 +endi +if $data60 != @22-04-26 15:15:07.000@ then + return -1 +endi +if $data61 != 1 then + return -1 +endi +if $data70 != @22-04-26 15:15:08.000@ then + return -1 +endi +if $data71 != 1 then + return -1 +endi +sql select * from (select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart) order by ts; +sql select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart; +sql select * from (select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart) order by ts; +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 interval(1s) order by _wstart); +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 partition by tag1 interval(1s) order by _wstart); + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 4db88e8ffe49e5dbbb56caf25b51d157dc13042e Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 15:45:58 +0800 Subject: [PATCH 029/187] fix: fix ci comments --- source/libs/executor/src/joinoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 700a59ccd8..9dee58367c 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -438,7 +438,6 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { - tSimpleHashClear(pInfo->rightBuildTable); SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen); From 99b424b181d527a3ce368b199dcf8be0a142d70d Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 16:26:20 +0800 Subject: [PATCH 030/187] fix: add test case --- tests/script/output.txt | 6 ++ tests/script/sh/l2norm2.c | 71 ++++++++++++++++++++ tests/script/tsim/parser/join_manyblocks.sim | 8 +-- 3 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 tests/script/output.txt create mode 100644 tests/script/sh/l2norm2.c diff --git a/tests/script/output.txt b/tests/script/output.txt new file mode 100644 index 0000000000..0c7c386bbc --- /dev/null +++ b/tests/script/output.txt @@ -0,0 +1,6 @@ +[03/30 08:08:52.140115] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection +[03/30 08:08:52.140136] ERROR: insert test process failed +[03/30 08:09:02.860424] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection +[03/30 08:09:02.860445] ERROR: insert test process failed +[03/30 08:10:57.082603] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection +[03/30 08:10:57.082626] ERROR: insert test process failed diff --git a/tests/script/sh/l2norm2.c b/tests/script/sh/l2norm2.c new file mode 100644 index 0000000000..0739f3a23b --- /dev/null +++ b/tests/script/sh/l2norm2.c @@ -0,0 +1,71 @@ +#include +#include +#include +#include + +#include "taosudf.h" + +DLL_EXPORT int32_t l2norm2_init() { + return 0; +} + +DLL_EXPORT int32_t l2norm2_destroy() { + return 0; +} + +DLL_EXPORT int32_t l2norm2_start(SUdfInterBuf *buf) { + *(int64_t*)(buf->buf) = 0; + buf->bufLen = sizeof(double); + buf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t l2norm2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { + double sumSquares = *(double*)interBuf->buf; + int8_t numNotNull = 0; + for (int32_t i = 0; i < block->numOfCols; ++i) { + SUdfColumn* col = block->udfCols[i]; + if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || + col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } + for (int32_t i = 0; i < block->numOfCols; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { + SUdfColumn* col = block->udfCols[i]; + if (udfColDataIsNull(col, j)) { + continue; + } + switch (col->colMeta.type) { + case TSDB_DATA_TYPE_INT: { + char* cell = udfColDataGetData(col, j); + int32_t num = *(int32_t*)cell; + sumSquares += (double)num * num; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + char* cell = udfColDataGetData(col, j); + double num = *(double*)cell; + sumSquares += num * num; + break; + } + default: + break; + } + ++numNotNull; + } + } + + *(double*)(newInterBuf->buf) = sumSquares; + newInterBuf->bufLen = sizeof(double); + newInterBuf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t l2norm2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { + double sumSquares = *(double*)(buf->buf); + *(double*)(resultData->buf) = sumSquares; + resultData->bufLen = sizeof(double); + resultData->numOfResult = 1; + return 0; +} diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index a40a75f50c..7fd0df21b3 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -6,8 +6,8 @@ sql connect $dbPrefix = join_m_db $tbPrefix = join_tb $mtPrefix = join_mt -$tbNum = 3 -$rowNum = 2000 +$tbNum = 20 +$rowNum = 200 $totalNum = $tbNum * $rowNum print =============== join_manyBlocks.sim @@ -78,8 +78,8 @@ print ==============> td-3313 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; print $row -if $row != 6000 then - print expect 6000, actual: $row +if $row != 4000 then + print expect 4000, actual: $row return -1 endi From 3e02bb8faa96dfb7c23fe34357d47231e1033dd9 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 16:26:52 +0800 Subject: [PATCH 031/187] fix: remove files --- tests/script/output.txt | 6 ---- tests/script/sh/l2norm2.c | 71 --------------------------------------- 2 files changed, 77 deletions(-) delete mode 100644 tests/script/output.txt delete mode 100644 tests/script/sh/l2norm2.c diff --git a/tests/script/output.txt b/tests/script/output.txt deleted file mode 100644 index 0c7c386bbc..0000000000 --- a/tests/script/output.txt +++ /dev/null @@ -1,6 +0,0 @@ -[03/30 08:08:52.140115] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection -[03/30 08:08:52.140136] ERROR: insert test process failed -[03/30 08:09:02.860424] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection -[03/30 08:09:02.860445] ERROR: insert test process failed -[03/30 08:10:57.082603] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection -[03/30 08:10:57.082626] ERROR: insert test process failed diff --git a/tests/script/sh/l2norm2.c b/tests/script/sh/l2norm2.c deleted file mode 100644 index 0739f3a23b..0000000000 --- a/tests/script/sh/l2norm2.c +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include -#include - -#include "taosudf.h" - -DLL_EXPORT int32_t l2norm2_init() { - return 0; -} - -DLL_EXPORT int32_t l2norm2_destroy() { - return 0; -} - -DLL_EXPORT int32_t l2norm2_start(SUdfInterBuf *buf) { - *(int64_t*)(buf->buf) = 0; - buf->bufLen = sizeof(double); - buf->numOfResult = 1; - return 0; -} - -DLL_EXPORT int32_t l2norm2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { - double sumSquares = *(double*)interBuf->buf; - int8_t numNotNull = 0; - for (int32_t i = 0; i < block->numOfCols; ++i) { - SUdfColumn* col = block->udfCols[i]; - if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || - col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) { - return TSDB_CODE_UDF_INVALID_INPUT; - } - } - for (int32_t i = 0; i < block->numOfCols; ++i) { - for (int32_t j = 0; j < block->numOfRows; ++j) { - SUdfColumn* col = block->udfCols[i]; - if (udfColDataIsNull(col, j)) { - continue; - } - switch (col->colMeta.type) { - case TSDB_DATA_TYPE_INT: { - char* cell = udfColDataGetData(col, j); - int32_t num = *(int32_t*)cell; - sumSquares += (double)num * num; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - char* cell = udfColDataGetData(col, j); - double num = *(double*)cell; - sumSquares += num * num; - break; - } - default: - break; - } - ++numNotNull; - } - } - - *(double*)(newInterBuf->buf) = sumSquares; - newInterBuf->bufLen = sizeof(double); - newInterBuf->numOfResult = 1; - return 0; -} - -DLL_EXPORT int32_t l2norm2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { - double sumSquares = *(double*)(buf->buf); - *(double*)(resultData->buf) = sumSquares; - resultData->bufLen = sizeof(double); - resultData->numOfResult = 1; - return 0; -} From 79623860a43cdd9604e7e41037bdb5db6044d393 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 May 2023 17:41:01 +0800 Subject: [PATCH 032/187] fix: distinct wrong result issue --- source/libs/planner/src/planLogicCreater.c | 9 ++++-- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/groupby_distinct.sim | 30 ++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 tests/script/tsim/query/groupby_distinct.sim diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b9ea5ba0cd..16bd16238e 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -544,11 +544,16 @@ static SNode* createGroupingSetNode(SNode* pExpr) { return (SNode*)pGroupingSet; } -static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static EGroupAction getDistinctGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; } +static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { + return ((pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) && !pSelect->isDistinct) ? GROUP_ACTION_KEEP + : GROUP_ACTION_NONE; +} + static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; @@ -1152,7 +1157,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = GROUP_ACTION_CLEAR; + pAgg->node.groupAction = getDistinctGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e79cf77648..bcfb392bb4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -900,6 +900,7 @@ ,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim +,,y,script,./test.sh -f tsim/query/groupby_distinct.sim ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim diff --git a/tests/script/tsim/query/groupby_distinct.sim b/tests/script/tsim/query/groupby_distinct.sim new file mode 100644 index 0000000000..8b16bb1910 --- /dev/null +++ b/tests/script/tsim/query/groupby_distinct.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + + +sql drop database if exists db1; +sql create database db1; +sql use db1; + +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql insert into tba1 values ('2022-04-26 15:15:08', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:07', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:06', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:05', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:03', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:02', 1, "d"); +sql insert into tba1 values ('2022-04-26 15:15:01', 1, "d"); +sql select distinct avg(f1) as avgv from sta group by f2; +if $rows != 1 then + return -1 +endi +sql select distinct avg(f1) as avgv from sta group by f2 limit 1,10; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From c7e6883ac8e2f47a2514246abecd675781f88f91 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 18:10:33 +0800 Subject: [PATCH 033/187] fix:tmq ref err & vg error --- source/client/src/clientTmq.c | 101 +++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 20 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index b488af9ba1..be547c42b2 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -87,6 +87,7 @@ struct tmq_t { void* commitCbUserParam; // status + SRWLatch lock; int8_t status; int32_t epoch; #if 0 @@ -156,6 +157,7 @@ typedef struct { int8_t tmqRspType; int32_t epoch; // epoch can be used to guard the vgHandle int32_t vgId; + char topicName[TSDB_TOPIC_FNAME_LEN]; SMqClientVg* vgHandle; SMqClientTopic* topicHandle; uint64_t reqId; @@ -168,8 +170,8 @@ typedef struct { } SMqPollRspWrapper; typedef struct { - int64_t refId; - int32_t epoch; +// int64_t refId; +// int32_t epoch; tsem_t rspSem; int32_t rspErr; } SMqSubscribeCbParam; @@ -184,8 +186,9 @@ typedef struct { typedef struct { int64_t refId; int32_t epoch; - SMqClientVg* pVg; - SMqClientTopic* pTopic; + char topicName[TSDB_TOPIC_FNAME_LEN]; +// SMqClientVg* pVg; +// SMqClientTopic* pTopic; int32_t vgId; uint64_t requestId; // request id for debug purpose } SMqPollCbParam; @@ -709,8 +712,8 @@ static void generateTimedTask(int64_t refId, int32_t type) { *pTaskType = type; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); } - taosReleaseRef(tmqMgmt.rsetId, refId); } void tmqAssignAskEpTask(void* param, void* tmrId) { @@ -1037,6 +1040,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->commitCb = conf->commitCb; pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; + taosInitRWLatch(&pTmq->lock); pTmq->hbBgEnable = conf->hbBgEnable; @@ -1140,8 +1144,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SMqSubscribeCbParam param = { .rspErr = 0, - .refId = tmq->refId, - .epoch = tmq->epoch, +// .refId = tmq->refId, +// .epoch = tmq->epoch, }; if (tsem_init(¶m.rspSem, 0, 0) != 0) { @@ -1217,12 +1221,40 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para conf->commitCbUserParam = param; } +static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + for (int32_t j = 0; j < vgNumCur; j++) { + SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); + if(pVgCur->vgId == vgId){ + return pVgCur; + } + } + } + } + return NULL; +} + +static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + return pTopicCur; + } + } + return NULL; +} + int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; int64_t refId = pParam->refId; - SMqClientVg* pVg = pParam->pVg; - SMqClientTopic* pTopic = pParam->pTopic; +// SMqClientVg* pVg = pParam->pVg; +// SMqClientTopic* pTopic = pParam->pTopic; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { @@ -1303,11 +1335,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } pRspWrapper->tmqRspType = rspType; - pRspWrapper->vgHandle = pVg; - pRspWrapper->topicHandle = pTopic; +// pRspWrapper->vgHandle = pVg; +// pRspWrapper->topicHandle = pTopic; pRspWrapper->reqId = requestId; pRspWrapper->pEpset = pMsg->pEpSet; - pRspWrapper->vgId = pVg->vgId; + pRspWrapper->vgId = vgId; + strcpy(pRspWrapper->topicName, pParam->topicName); pMsg->pEpSet = NULL; if (rspType == TMQ_MSG_TYPE__POLL_RSP) { @@ -1351,7 +1384,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { CREATE_MSG_FAIL: if (epoch == tmq->epoch) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = getVgInfo(tmq, pParam->topicName, vgId); + if(pVg){ + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + taosWLockLatch(&tmq->lock); } tsem_post(&tmq->rspSem); @@ -1472,11 +1510,13 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) taosHashCleanup(pVgOffsetHashMap); + taosWLockLatch(&tmq->lock); // destroy current buffered existed topics info if (tmq->clientTopics) { taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo); } tmq->clientTopics = newTopics; + taosWUnLockLatch(&tmq->lock); int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; atomic_store_8(&tmq->status, flag); @@ -1492,7 +1532,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { if (tmq == NULL) { terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); +// pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -1652,8 +1692,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam->refId = pTmq->refId; pParam->epoch = pTmq->epoch; - pParam->pVg = pVg; // pVg may be released,fix it - pParam->pTopic = pTopic; +// pParam->pVg = pVg; // pVg may be released,fix it +// pParam->pTopic = pTopic; + strcpy(pParam->topicName, pTopic->topicName); pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; @@ -1779,8 +1820,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; if (pDataRsp->head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; - + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } // update the epset if (pollRspWrapper->pEpset != NULL) { SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset); @@ -1829,7 +1876,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } if(pollRspWrapper->metaRsp.rspOffset.type != 0){ // if offset is validate pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; } @@ -1849,7 +1903,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate pVg->currentOffset = pollRspWrapper->taosxRsp.rspOffset; } From 7db9f1e58ed471bcd22e5d692bd11c47835140d6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 18:21:45 +0800 Subject: [PATCH 034/187] fix:tmq ref err & vg error --- source/client/src/clientTmq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index be547c42b2..433b27d5d9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1269,8 +1269,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t vgId = pParam->vgId; uint64_t requestId = pParam->requestId; - taosMemoryFree(pParam); - if (code != 0) { if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); @@ -1314,6 +1312,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pParam); + return 0; } @@ -1379,6 +1379,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return 0; @@ -1394,6 +1395,7 @@ CREATE_MSG_FAIL: tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return -1; } From 50bdfef7b6e32134a928fc7e6cd0232b2163054e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 18:34:42 +0800 Subject: [PATCH 035/187] fix:tmq ref err & vg error --- source/client/src/clientTmq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 433b27d5d9..01667f6dc3 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1390,7 +1390,7 @@ CREATE_MSG_FAIL: if(pVg){ atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } - taosWLockLatch(&tmq->lock); + taosWUnLockLatch(&tmq->lock); } tsem_post(&tmq->rspSem); From bca1af3f5d7cd00d98096b1b0f5e052b8e8c0790 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 21:18:44 +0800 Subject: [PATCH 036/187] fix:change tmq subscribe try time to 60mins --- source/client/src/clientTmq.c | 4 ++-- source/dnode/vnode/src/tq/tqPush.c | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 01667f6dc3..c08fbd0adf 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1089,7 +1089,7 @@ _failed: } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { - const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most + const int32_t MAX_RETRY_COUNT = 120 * 60; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1186,7 +1186,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { if (retryCnt++ > MAX_RETRY_COUNT) { tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); - code = TSDB_CODE_TSC_INTERNAL_ERROR; + code = TSDB_CODE_MND_CONSUMER_NOT_READY; goto FAIL; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 50d94c5ed5..42c60e0007 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -73,8 +73,11 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) { STqHandle *pHandle = (STqHandle*)handle; int32_t vgId = TD_VID(pTq->pVnode); + if(taosHashGetSize(pTq->pPushMgr) <= 0) { + return 0; + } int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey)); - tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); + tqDebug("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); if(pHandle->msg != NULL) { tqPushDataRsp(pTq, pHandle); From d4b0e102af08e156ca8bf61fa091b38d21ca3933 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 May 2023 08:55:59 +0800 Subject: [PATCH 037/187] fix:do some internal refactor. --- include/common/ttime.h | 2 +- source/common/src/ttime.c | 2 -- source/libs/executor/src/executil.c | 2 +- source/libs/parser/src/parTranslater.c | 8 ++++---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 69a071d59f..7bbc6a8e56 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -23,7 +23,7 @@ extern "C" { #endif -#define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') +#define IS_CALENDAR_TIME_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') #define TIME_UNIT_NANOSECOND 'b' #define TIME_UNIT_MICROSECOND 'u' diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index de6a18f7c8..165a4ff95b 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -709,8 +709,6 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati return getDuration(*duration, *unit, duration, timePrecision); } -#define IS_CALENDAR_TIME_DURATION(_d) (((_d) == 'n') || ((_d) == 'y')) - int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { return t; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 91f384ce6a..b4c5a90be3 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1759,7 +1759,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->slidingUnit != 'n' && pInterval->slidingUnit != 'y') { + if (!IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { tw->skey += pInterval->sliding * factor; tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0d052846f7..c61c4fe4d4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3004,7 +3004,7 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey); int64_t intervalRange = 0; - if (TIME_IS_VAR_DURATION(pInterval->unit)) { + if (IS_CALENDAR_TIME_DURATION(pInterval->unit)) { int64_t f = 1; if (pInterval->unit == 'n') { f = 30LL * MILLISECOND_PER_DAY; @@ -3072,7 +3072,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = TIME_IS_VAR_DURATION(pInter->unit); + bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit); if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, getPrecisionStr(precision)); @@ -3086,7 +3086,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->unit == 'n' && pOffset->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter; + bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { @@ -3098,7 +3098,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* const static int32_t INTERVAL_SLIDING_FACTOR = 100; SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (TIME_IS_VAR_DURATION(pSliding->unit)) { + if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || From 93c4e3fe70a9a989d8b96a457b5240147190b4a7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 May 2023 09:12:16 +0800 Subject: [PATCH 038/187] other: remove empty files. --- source/libs/executor/inc/executorimpl.h | 0 source/libs/executor/src/executorimpl.c | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 source/libs/executor/inc/executorimpl.h delete mode 100644 source/libs/executor/src/executorimpl.c diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c deleted file mode 100644 index e69de29bb2..0000000000 From e882601b67f900dde0790d1e55628dfd7b86e4c5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 May 2023 09:15:48 +0800 Subject: [PATCH 039/187] fix: last table scan issue --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 21b59e376c..4acc784aee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3112,6 +3112,10 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) { + // reset the index in last block when handing a new file + doCleanupTableScanInfo(pScanInfo); + pStatus->mapDataCleaned = true; + bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { return TSDB_CODE_SUCCESS; From 7218a034870adc3f336b4672b88e12639b20daa2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 May 2023 11:16:31 +0800 Subject: [PATCH 040/187] fix: result block ensure capacity when finalize may produce serveral rows --- source/libs/executor/src/executorInt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 1a14d0e78a..303be3d2cd 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -715,7 +715,8 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR pCtx[j].resultInfo->numOfRes = pRow->numOfRows; } } - + + blockDataEnsureCapacity(pBlock, pBlock->info.rows + pCtx[j].resultInfo->numOfRes); int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); if (TAOS_FAILED(code)) { qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); From 6a1eb62dd44956c6cb4a7453c83eb96687e1cefc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 May 2023 11:26:40 +0800 Subject: [PATCH 041/187] fix: distinct group issue --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 16bd16238e..1ec0abe8df 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1157,7 +1157,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = getDistinctGroupAction(pCxt, pSelect); + pAgg->node.groupAction = GROUP_ACTION_CLEAR;//getDistinctGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; From 238254e49ee83723dd274bae5a66d7fe9994d5df Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 14:06:43 +0800 Subject: [PATCH 042/187] fix:add task status check in doQueueScan & checkout consumer in tqProcessPollReq --- source/dnode/vnode/src/tq/tq.c | 22 +++++++++--------- source/libs/executor/src/scanoperator.c | 30 +------------------------ 2 files changed, 11 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e074a2d8a6..41b43026ea 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -358,9 +358,19 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } + // 2. check re-balance status + if (pHandle->consumerId != consumerId) { + tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosWUnLockLatch(&pTq->lock); + return -1; + } + bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; } @@ -370,17 +380,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } - // 2. check re-balance status - if (pHandle->consumerId != consumerId) { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, - consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - code = -1; - goto end; - } - - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); - // 3. update the epoch value int32_t savedEpoch = pHandle->epoch; if (savedEpoch < reqEpoch) { @@ -396,7 +395,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); -end: tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7d3165237e..638662aeb5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1665,37 +1665,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("start to exec queue scan, %s", id); -#if 0 - if (pTaskInfo->streamInfo.submit.msgStr != NULL) { - if (pInfo->tqReader->msg.msgStr == NULL) { - SPackedData submit = pTaskInfo->streamInfo.submit; - if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { - qError("submit msg messed up when initing stream submit block %p", submit.msgStr); - return NULL; - } - } - - blockDataCleanup(pInfo->pRes); - SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - - while (tqNextBlockImpl(pInfo->tqReader)) { - int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL); - if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { - continue; - } - - setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true); - - if (pBlockInfo->rows > 0) { - return pInfo->pRes; - } - } - - pInfo->tqReader->msg = (SPackedData){0}; - pTaskInfo->streamInfo.submit = (SPackedData){0}; + if (isTaskKilled(pTaskInfo)) { return NULL; } -#endif if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); From f36b0be17d0c91dcd45941bd5d753d39293e6856 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 May 2023 14:32:03 +0800 Subject: [PATCH 043/187] fix: join eq conditions not only for tag --- include/libs/nodes/plannodes.h | 16 +++---- source/libs/executor/src/joinoperator.c | 56 +++++++++++----------- source/libs/nodes/src/nodesCloneFuncs.c | 4 +- source/libs/nodes/src/nodesCodeFuncs.c | 12 ++--- source/libs/nodes/src/nodesMsgFuncs.c | 8 ++-- source/libs/nodes/src/nodesUtilFuncs.c | 14 +++--- source/libs/planner/src/planOptimizer.c | 33 ++++++------- source/libs/planner/src/planPhysiCreater.c | 6 +-- 8 files changed, 73 insertions(+), 76 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 0e235191b1..f559ba2b51 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,7 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; - SNode* pTagEqualConditions; + SNode* pEqualOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -406,7 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; - SNode* pTagEqualCondtions; + SNode* pEqualOnCondtions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { @@ -448,7 +448,7 @@ typedef struct SMergePhysiNode { bool groupSort; } SMergePhysiNode; -typedef struct SWinodwPhysiNode { +typedef struct SWindowPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; @@ -461,10 +461,10 @@ typedef struct SWinodwPhysiNode { EOrder inputTsOrder; EOrder outputTsOrder; bool mergeDataBlock; -} SWinodwPhysiNode; +} SWindowPhysiNode; typedef struct SIntervalPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t interval; int64_t offset; int64_t sliding; @@ -497,7 +497,7 @@ typedef struct SMultiTableIntervalPhysiNode { } SMultiTableIntervalPhysiNode; typedef struct SSessionWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t gap; } SSessionWinodwPhysiNode; @@ -506,14 +506,14 @@ typedef SSessionWinodwPhysiNode SStreamSemiSessionWinodwPhysiNode; typedef SSessionWinodwPhysiNode SStreamFinalSessionWinodwPhysiNode; typedef struct SStateWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStateKey; } SStateWinodwPhysiNode; typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; typedef struct SEventWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStartCond; SNode* pEndCond; } SEventWinodwPhysiNode; diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 9dee58367c..46c6b24295 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -52,15 +52,15 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; - SNode* pTagEqualConditions; + SNode* pEqualOnConditions; - SArray* leftTagCols; - char* leftTagKeyBuf; - int32_t leftTagKeyLen; + SArray* leftEqOnCondCols; + char* leftEqOnCondKeyBuf; + int32_t leftEqOnCondKeyLen; - SArray* rightTagCols; - char* rightTagKeyBuf; - int32_t rightTagKeyLen; + SArray* rightEqOnCondCols; + char* rightEqOnCondKeyBuf; + int32_t rightEqOnCondKeyLen; SSHashObj* rightBuildTable; SJoinRowCtx rowCtx; @@ -104,7 +104,7 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } -static void extractTagEqualColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, +static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, SColumn* pLeft, SColumn* pRight) { SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft; SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight; @@ -125,7 +125,7 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD SNode* pNode = NULL; FOREACH(pNode, ((SLogicConditionNode*)pTagEqualNode)->pParameterList) { SOperatorNode* pOperNode = (SOperatorNode*)pNode; - extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); taosArrayPush(rightTagEqCols, &right); } @@ -134,7 +134,7 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD if (nodeType(pTagEqualNode) == QUERY_NODE_OPERATOR) { SOperatorNode* pOperNode = (SOperatorNode*)pTagEqualNode; - extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); taosArrayPush(rightTagEqCols, &right); } @@ -259,13 +259,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pInfo->pTagEqualConditions = pJoinNode->pTagEqualCondtions; - if (pInfo->pTagEqualConditions != NULL) { - pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); - pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); - extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); - initTagColskeyBuf(&pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); - initTagColskeyBuf(&pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + pInfo->pEqualOnConditions = pJoinNode->pEqualOnCondtions; + if (pInfo->pEqualOnConditions != NULL) { + pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + extractTagEqualCondCols(pInfo, pDownstream, pInfo->pEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); + initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols); + initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->rightBuildTable = tSimpleHashInit(256, hashFn); } @@ -309,13 +309,13 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; - if (pJoinOperator->pTagEqualConditions != NULL) { + if (pJoinOperator->pEqualOnConditions != NULL) { mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); - taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); - taosArrayDestroy(pJoinOperator->rightTagCols); + taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->rightEqOnCondCols); - taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); - taosArrayDestroy(pJoinOperator->leftTagCols); + taosMemoryFreeClear(pJoinOperator->leftEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->leftEqOnCondCols); } nodesDestroyNode(pJoinOperator->pCondAfterMerge); @@ -439,12 +439,12 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); - int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); - SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen); + int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightEqOnCondCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightEqOnCondKeyBuf); + SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen); if (!ppRows) { SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); taosArrayPush(rows, rightRow); - tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); + tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen, &rows, POINTER_BYTES); } else { taosArrayPush(*ppRows, rightRow); } @@ -466,8 +466,8 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SArray* pRightRows = NULL; if (useBuildTableTSRange) { - int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftTagCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftTagKeyBuf); - SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftTagKeyBuf, keyLen); + int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftEqOnCondCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftEqOnCondKeyBuf); + SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftEqOnCondKeyBuf, keyLen); if (!ppRightRows) { continue; } @@ -567,7 +567,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); - if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + if (pJoinInfo->pEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 91db527a02..c2771e0005 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,7 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); - CLONE_NODE_FIELD(pTagEqualConditions); + CLONE_NODE_FIELD(pEqualOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; @@ -588,7 +588,7 @@ static int32_t physiSysTableScanCopy(const SSystemTableScanPhysiNode* pSrc, SSys return TSDB_CODE_SUCCESS; } -static int32_t physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* pDst) { +static int32_t physiWindowCopy(const SWindowPhysiNode* pSrc, SWindowPhysiNode* pDst) { COPY_BASE_OBJECT_FIELD(node, physiNodeCopy); CLONE_NODE_LIST_FIELD(pExprs); CLONE_NODE_LIST_FIELD(pFuncs); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b4e2d95e26..d16cd79e97 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1432,7 +1432,7 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pTagEqualConditions); + code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pEqualOnConditions); } return code; } @@ -1451,7 +1451,7 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pTagEqualConditions); + code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pEqualOnConditions); } return code; } @@ -1905,7 +1905,7 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pTagEqualCondtions); + code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pEqualOnCondtions); } return code; } @@ -1930,7 +1930,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pTagEqualCondtions); + code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pEqualOnCondtions); } return code; } @@ -2135,7 +2135,7 @@ static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { @@ -2176,7 +2176,7 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { } static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 45cebb4559..59b027d5ed 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2341,7 +2341,7 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pTagEqualCondtions); + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pEqualOnCondtions); } return code; } @@ -2372,7 +2372,7 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: - code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagEqualCondtions); + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEqualOnCondtions); break; default: break; @@ -2639,7 +2639,7 @@ enum { }; static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); if (TSDB_CODE_SUCCESS == code) { @@ -2680,7 +2680,7 @@ static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { } static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = TSDB_CODE_SUCCESS; STlv* pTlv = NULL; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 67747497db..81b429e169 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -599,7 +599,7 @@ static void destroyPhysiNode(SPhysiNode* pNode) { nodesDestroyNode(pNode->pSlimit); } -static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { +static void destroyWinodwPhysiNode(SWindowPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pExprs); nodesDestroyList(pNode->pFuncs); @@ -1072,7 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); - nodesDestroyNode(pLogicNode->pTagEqualConditions); + nodesDestroyNode(pLogicNode->pEqualOnConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1205,7 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); - nodesDestroyNode(pPhyNode->pTagEqualCondtions); + nodesDestroyNode(pPhyNode->pEqualOnCondtions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { @@ -1243,7 +1243,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: { @@ -1259,19 +1259,19 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: { SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStateKey); break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStartCond); nodesDestroyNode(pPhyNode->pEndCond); break; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 5be67389c8..1107389df9 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -740,18 +740,15 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin return code; } -static bool pushDownCondOptIsTag(SNode* pNode, SNodeList* pTableCols) { +static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) { if (QUERY_NODE_COLUMN != nodeType(pNode)) { return false; } SColumnNode* pCol = (SColumnNode*)pNode; - if (COLUMN_TYPE_TAG != pCol->colType) { - return false; - } return pushDownCondOptBelongThisTable(pNode, pTableCols); } -static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } @@ -770,22 +767,22 @@ static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { } SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; - if (pushDownCondOptIsTag(pOper->pLeft, pLeftCols)) { - return pushDownCondOptIsTag(pOper->pRight, pRightCols); - } else if (pushDownCondOptIsTag(pOper->pLeft, pRightCols)) { - return pushDownCondOptIsTag(pOper->pRight, pLeftCols); + if (pushDownCondOptIsTableColumn(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsTableColumn(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pLeftCols); } return false; } -static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); int32_t code = TSDB_CODE_SUCCESS; SNodeList* pTagEqualConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - if (pushDownCondOptIsTagEqualCond(pJoin, pCond)) { + if (pushDownCondOptIsEqualOnCond(pJoin, pCond)) { code = nodesListMakeAppend(&pTagEqualConds, nodesCloneNode(pCond)); } } @@ -796,7 +793,7 @@ static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin } if (TSDB_CODE_SUCCESS == code) { - pJoin->pTagEqualConditions = pTempTagEqCond; + pJoin->pEqualOnConditions = pTempTagEqCond; return TSDB_CODE_SUCCESS; } else { nodesDestroyList(pTagEqualConds); @@ -805,18 +802,18 @@ static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin return TSDB_CODE_SUCCESS; } -static int32_t pushDownCondOptJoinExtractTagEqualCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { - pJoin->pTagEqualConditions = NULL; + pJoin->pEqualOnConditions = NULL; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { - return pushDownCondOptJoinExtractTagEqualLogicCond(pJoin); + return pushDownCondOptJoinExtractEqualOnLogicCond(pJoin); } - if (pushDownCondOptIsTagEqualCond(pJoin, pJoin->pOnConditions)) { - pJoin->pTagEqualConditions = nodesCloneNode(pJoin->pOnConditions); + if (pushDownCondOptIsEqualOnCond(pJoin, pJoin->pOnConditions)) { + pJoin->pEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); } return TSDB_CODE_SUCCESS; @@ -857,7 +854,7 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p } if (TSDB_CODE_SUCCESS == code) { - code = pushDownCondOptJoinExtractTagEqualCond(pCxt, pJoin); + code = pushDownCondOptJoinExtractEqualOnCond(pCxt, pJoin); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e07c6ebcfe..a13ca6cba9 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,8 +705,8 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } - if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pTagEqualConditions) { - code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pTagEqualConditions, &pJoin->pTagEqualCondtions); + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pEqualOnConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pEqualOnConditions, &pJoin->pEqualOnCondtions); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); @@ -1150,7 +1150,7 @@ static int32_t createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNo } } -static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWinodwPhysiNode* pWindow, +static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowPhysiNode* pWindow, SWindowLogicNode* pWindowLogicNode) { pWindow->triggerType = pWindowLogicNode->triggerType; pWindow->watermark = pWindowLogicNode->watermark; From f7ab8dabf086ef3740040635059aa36627bc1525 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 15:14:39 +0800 Subject: [PATCH 044/187] fix:[TD-23972] push subscribe msg to vnode even though consumer not change --- source/dnode/mnode/impl/src/mndSubscribe.c | 21 ++++++++++++---- source/dnode/vnode/src/tq/tq.c | 28 ++++++++++++---------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 2ae4ede25a..e62102fa77 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -133,10 +133,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg) { - if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { - terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; - return -1; - } +// if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { +// terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; +// return -1; +// } void *buf; int32_t tlen; @@ -269,6 +269,18 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { } } +static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ + for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ + SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); + SMqRebOutputVg outputVg = { + .oldConsumerId = pConsumerEp->consumerId, + .newConsumerId = pConsumerEp->consumerId, + .pVgEp = pVgEp, + }; + taosArrayPush(pOutput->rebVgs, &outputVg); + } +} + static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, int32_t imbConsumerNum) { const char *pSubKey = pOutput->pSub->key; @@ -318,6 +330,7 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas } } } + putNoTransferToOutput(pOutput, pConsumerEp); } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 41b43026ea..6f619e9bbe 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -569,20 +569,22 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); // atomic_store_32(&pHandle->epoch, 0); - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } - // remove if it has been register in the push manager, and return one empty block to consumer - taosWLockLatch(&pTq->lock); - tqUnregisterPushHandle(pTq, pHandle); - taosWUnLockLatch(&pTq->lock); - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } + + // kill executing task + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + } + + taosWLockLatch(&pTq->lock); + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } + // remove if it has been register in the push manager, and return one empty block to consumer + tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } end: From f9da4abdcb495d0d9c75e4a6e9f14da47621a070 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 May 2023 15:38:52 +0800 Subject: [PATCH 045/187] fix: change more variable names --- include/libs/nodes/plannodes.h | 4 ++-- source/libs/executor/src/joinoperator.c | 22 +++++++++++----------- source/libs/nodes/src/nodesCloneFuncs.c | 2 +- source/libs/nodes/src/nodesCodeFuncs.c | 12 ++++++------ source/libs/nodes/src/nodesMsgFuncs.c | 4 ++-- source/libs/nodes/src/nodesUtilFuncs.c | 4 ++-- source/libs/planner/src/planOptimizer.c | 14 +++++++------- source/libs/planner/src/planPhysiCreater.c | 4 ++-- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f559ba2b51..a27237cab3 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,7 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; - SNode* pEqualOnConditions; + SNode* pColEqualOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -406,7 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; - SNode* pEqualOnCondtions; + SNode* pColEqualOnConditions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 46c6b24295..442f8162ed 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -52,7 +52,7 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; - SNode* pEqualOnConditions; + SNode* pColEqualOnConditions; SArray* leftEqOnCondCols; char* leftEqOnCondKeyBuf; @@ -117,13 +117,13 @@ static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorIn } } -static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pTagEqualNode, +static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pEqualOnCondNode, SArray* leftTagEqCols, SArray* rightTagEqCols) { SColumn left = {0}; SColumn right = {0}; - if (nodeType(pTagEqualNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pTagEqualNode)->condType == LOGIC_COND_TYPE_AND) { + if (nodeType(pEqualOnCondNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pEqualOnCondNode)->condType == LOGIC_COND_TYPE_AND) { SNode* pNode = NULL; - FOREACH(pNode, ((SLogicConditionNode*)pTagEqualNode)->pParameterList) { + FOREACH(pNode, ((SLogicConditionNode*)pEqualOnCondNode)->pParameterList) { SOperatorNode* pOperNode = (SOperatorNode*)pNode; extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); @@ -132,8 +132,8 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD return; } - if (nodeType(pTagEqualNode) == QUERY_NODE_OPERATOR) { - SOperatorNode* pOperNode = (SOperatorNode*)pTagEqualNode; + if (nodeType(pEqualOnCondNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pOperNode = (SOperatorNode*)pEqualOnCondNode; extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); taosArrayPush(rightTagEqCols, &right); @@ -259,11 +259,11 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pInfo->pEqualOnConditions = pJoinNode->pEqualOnCondtions; - if (pInfo->pEqualOnConditions != NULL) { + pInfo->pColEqualOnConditions = pJoinNode->pColEqualOnConditions; + if (pInfo->pColEqualOnConditions != NULL) { pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); - extractTagEqualCondCols(pInfo, pDownstream, pInfo->pEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); + extractEqualOnCondCols(pInfo, pDownstream, pInfo->pColEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols); initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -309,7 +309,7 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; - if (pJoinOperator->pEqualOnConditions != NULL) { + if (pJoinOperator->pColEqualOnConditions != NULL) { mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf); taosArrayDestroy(pJoinOperator->rightEqOnCondCols); @@ -567,7 +567,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); - if (pJoinInfo->pEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + if (pJoinInfo->pColEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index c2771e0005..f18e666741 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,7 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); - CLONE_NODE_FIELD(pEqualOnConditions); + CLONE_NODE_FIELD(pColEqualOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index d16cd79e97..f061fe60d2 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1416,7 +1416,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanMergeCondition = "MergeConditions"; -static const char* jkJoinLogicPlanTagEqualConditions = "TagEqualConditions"; +static const char* jkJoinLogicPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; @@ -1432,7 +1432,7 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pEqualOnConditions); + code = tjsonAddObject(pJson, jkJoinLogicPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); } return code; } @@ -1451,7 +1451,7 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pEqualOnConditions); + code = jsonToNodeObject(pJson, jkJoinLogicPlanColEqualOnConditions, &pNode->pColEqualOnConditions); } return code; } @@ -1883,7 +1883,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition"; static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; -static const char* jkJoinPhysiPlanTagEqualConditions = "TagEqualConditions"; +static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; @@ -1905,7 +1905,7 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pEqualOnCondtions); + code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); } return code; } @@ -1930,7 +1930,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pEqualOnCondtions); + code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqualOnConditions); } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 59b027d5ed..e200b5dac5 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2341,7 +2341,7 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pEqualOnCondtions); + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqualOnConditions); } return code; } @@ -2372,7 +2372,7 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: - code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEqualOnCondtions); + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqualOnConditions); break; default: break; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 81b429e169..830d5886f4 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1072,7 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); - nodesDestroyNode(pLogicNode->pEqualOnConditions); + nodesDestroyNode(pLogicNode->pColEqualOnConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1205,7 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); - nodesDestroyNode(pPhyNode->pEqualOnCondtions); + nodesDestroyNode(pPhyNode->pColEqualOnConditions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 1107389df9..647a55a495 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -779,24 +779,24 @@ static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); int32_t code = TSDB_CODE_SUCCESS; - SNodeList* pTagEqualConds = NULL; + SNodeList* pEqualOnConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { if (pushDownCondOptIsEqualOnCond(pJoin, pCond)) { - code = nodesListMakeAppend(&pTagEqualConds, nodesCloneNode(pCond)); + code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond)); } } SNode* pTempTagEqCond = NULL; if (TSDB_CODE_SUCCESS == code) { - code = nodesMergeConds(&pTempTagEqCond, &pTagEqualConds); + code = nodesMergeConds(&pTempTagEqCond, &pEqualOnConds); } if (TSDB_CODE_SUCCESS == code) { - pJoin->pEqualOnConditions = pTempTagEqCond; + pJoin->pColEqualOnConditions = pTempTagEqCond; return TSDB_CODE_SUCCESS; } else { - nodesDestroyList(pTagEqualConds); + nodesDestroyList(pEqualOnConds); return TSDB_CODE_PLAN_INTERNAL_ERROR; } return TSDB_CODE_SUCCESS; @@ -804,7 +804,7 @@ static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { - pJoin->pEqualOnConditions = NULL; + pJoin->pColEqualOnConditions = NULL; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && @@ -813,7 +813,7 @@ static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJo } if (pushDownCondOptIsEqualOnCond(pJoin, pJoin->pOnConditions)) { - pJoin->pEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); + pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index a13ca6cba9..f11f1244b4 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,8 +705,8 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } - if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pEqualOnConditions) { - code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pEqualOnConditions, &pJoin->pEqualOnCondtions); + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqualOnConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqualOnConditions, &pJoin->pColEqualOnConditions); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); From d77eefdc93c7d6584eb033fb588b45714db66433 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 May 2023 15:43:31 +0800 Subject: [PATCH 046/187] fix: change more variable names --- source/libs/planner/src/planOptimizer.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 647a55a495..80beab3038 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -748,7 +748,7 @@ static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) { return pushDownCondOptBelongThisTable(pNode, pTableCols); } -static bool pushDownCondOptIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptIsColEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } @@ -775,14 +775,14 @@ static bool pushDownCondOptIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { return false; } -static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractColEqualOnLogicCond(SJoinLogicNode* pJoin) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); int32_t code = TSDB_CODE_SUCCESS; SNodeList* pEqualOnConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - if (pushDownCondOptIsEqualOnCond(pJoin, pCond)) { + if (pushDownCondOptIsColEqualOnCond(pJoin, pCond)) { code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond)); } } @@ -802,17 +802,17 @@ static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) return TSDB_CODE_SUCCESS; } -static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractColEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { pJoin->pColEqualOnConditions = NULL; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { - return pushDownCondOptJoinExtractEqualOnLogicCond(pJoin); + return pushDownCondOptJoinExtractColEqualOnLogicCond(pJoin); } - if (pushDownCondOptIsEqualOnCond(pJoin, pJoin->pOnConditions)) { + if (pushDownCondOptIsColEqualOnCond(pJoin, pJoin->pOnConditions)) { pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); } @@ -854,7 +854,7 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p } if (TSDB_CODE_SUCCESS == code) { - code = pushDownCondOptJoinExtractEqualOnCond(pCxt, pJoin); + code = pushDownCondOptJoinExtractColEqualOnCond(pCxt, pJoin); } if (TSDB_CODE_SUCCESS == code) { From be35194d169e77e97f524ff9e48e3c6c701fa58e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 May 2023 15:54:46 +0800 Subject: [PATCH 047/187] fix(query): add null check. --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index a196c2d398..8f168e67a8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -225,6 +225,9 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, void* tsdbCacherowsReaderClose(void* pReader) { SCacheRowsReader* p = pReader; + if (p == NULL) { + return NULL; + } if (p->pSchema != NULL) { for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) { From 6e6853413c0e4ce57dcd9355628cea2dc72b2fac Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 May 2023 16:48:05 +0800 Subject: [PATCH 048/187] fix: db cfg memory issue --- source/dnode/mnode/impl/src/mndDb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e93b06fdea..0f7e3de2f5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -929,7 +929,7 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->walRetentionSize = pDb->cfg.walRetentionSize; cfgRsp->walSegmentSize = pDb->cfg.walSegmentSize; cfgRsp->numOfRetensions = pDb->cfg.numOfRetensions; - cfgRsp->pRetensions = pDb->cfg.pRetensions; + cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); cfgRsp->schemaless = pDb->cfg.schemaless; cfgRsp->sstTrigger = pDb->cfg.sstTrigger; } @@ -972,6 +972,8 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { _OVER: + tFreeSDbCfgRsp(&cfgRsp); + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } From 3dbbe6309248c10e28cd7c2097f9d61f04d82b35 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 19 May 2023 16:50:27 +0800 Subject: [PATCH 049/187] fix: select last(*) ..group by tag return wrong rows of data --- source/libs/planner/src/planOptimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 5be67389c8..5d0757883c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2377,7 +2377,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (NULL != cxt.pLastCols) { cxt.doAgg = false; lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols); - NODES_DESTORY_LIST(pScan->pScanPseudoCols); + nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt); lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols); nodesClearList(cxt.pLastCols); } From ff2b545b27afa6f28f704d896910c04e727d19c8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 17:16:47 +0800 Subject: [PATCH 050/187] fix:set task status killed when vnode receive subscribe msg from mnode --- include/libs/executor/executor.h | 2 ++ source/dnode/vnode/src/tq/tq.c | 4 ++-- source/libs/executor/src/executor.c | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index b7e6c42e3b..2598b5c28c 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,6 +90,8 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); + int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); // todo refactor diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6f619e9bbe..9e43c4a944 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -394,7 +394,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); - + qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; @@ -574,7 +574,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // kill executing task qTaskInfo_t pTaskInfo = pHandle->execHandle.task; if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); } taosWLockLatch(&pTq->lock); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index c4a56d78ae..eea542e042 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,6 +180,11 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { + SExecTaskInfo* pTaskInfo = tinfo; + pTaskInfo->code = code; +} + int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { return TSDB_CODE_APP_ERROR; From c7c255e967aa250741bcb839f329634a99834b29 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 18:06:05 +0800 Subject: [PATCH 051/187] fix:set task status killed when vnode receive subscribe msg from mnode --- source/dnode/vnode/src/tq/tq.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9e43c4a944..b1e412d5ec 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -370,6 +370,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); + qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; @@ -394,7 +395,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); - qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; @@ -571,15 +571,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // atomic_store_32(&pHandle->epoch, 0); } - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); - } - taosWLockLatch(&pTq->lock); - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); + // kill executing task + if(tqIsHandleExec(pHandle)) { + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); + } + + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); From 33d6df8717875f1f2001d8ea8dde8339beee279d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 19:11:49 +0800 Subject: [PATCH 052/187] fix:set task status killed when vnode receive subscribe msg from mnode --- source/dnode/vnode/src/tq/tq.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b1e412d5ec..d9961202b1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -367,6 +367,20 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } + // 3. update the epoch value + int32_t savedEpoch = pHandle->epoch; + if (savedEpoch <= reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, + reqEpoch); + pHandle->epoch = reqEpoch; + }else { + tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", + consumerId, TD_VID(pTq->pVnode), req.subKey, savedEpoch, reqEpoch); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosWUnLockLatch(&pTq->lock); + return -1; + } + bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); @@ -381,14 +395,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } - // 3. update the epoch value - int32_t savedEpoch = pHandle->epoch; - if (savedEpoch < reqEpoch) { - tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, - reqEpoch); - pHandle->epoch = reqEpoch; - } - char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, @@ -396,6 +402,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); tqSetHandleIdle(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; } @@ -561,17 +568,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { + taosWLockLatch(&pTq->lock); + if (pHandle->consumerId == req.newConsumerId) { // do nothing tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs, should not reach here", req.vgId, req.newConsumerId); -// atomic_add_fetch_32(&pHandle->epoch, 1); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); -// atomic_store_32(&pHandle->epoch, 0); } + atomic_add_fetch_32(&pHandle->epoch, 1); - taosWLockLatch(&pTq->lock); // kill executing task if(tqIsHandleExec(pHandle)) { qTaskInfo_t pTaskInfo = pHandle->execHandle.task; From 1ddbdad33cf487e94d2edf6a3138324f123723e5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 21:40:33 +0800 Subject: [PATCH 053/187] fix:set task status killed when vnode receive subscribe msg from mnode --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d9961202b1..da366c1610 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -362,7 +362,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (pHandle->consumerId != consumerId) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + terrno = TSDB_CODE_TMQ_INVALID_MSG; taosWUnLockLatch(&pTq->lock); return -1; } From f8de38e5327ca92025c6f7a00fa6e02c08b6e710 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 22:52:58 +0800 Subject: [PATCH 054/187] fix:core dump in tsdbreader is null when tsdbreader is closed by subscribe msg and current offset equal to saved --- source/libs/executor/src/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index eea542e042..7d251fb074 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1080,7 +1080,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset) && pOffset->type != TMQ_OFFSET__SNAPSHOT_DATA) { return 0; } From 04858fae22b4b2f0fa6207b56e84dab0fc679ea6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 May 2023 12:20:54 +0800 Subject: [PATCH 055/187] fix:tsdbreader is free by mistake --- include/libs/executor/executor.h | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/dnode/vnode/src/tq/tq.c | 8 ++++---- source/dnode/vnode/src/tq/tqScan.c | 12 ++++++++---- source/libs/executor/src/executor.c | 10 +++++----- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 2598b5c28c..91db78e9cd 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,7 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index a318b9886e..922a85c094 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -75,7 +75,7 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, terrstr(code)); + dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, tstrerror(code)); vmSendRsp(pMsg, code); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index da366c1610..ce52277d71 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -384,7 +384,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); - qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); +// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; @@ -586,9 +586,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } +// if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { +// qStreamCloseTsdbReader(pTaskInfo); +// } } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index e4b2fa8821..2f470e2221 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -84,8 +84,10 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs qStreamSetOpen(task); tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); - if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { - tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); + code = qExecTask(task, &pDataBlock, &ts); + if (code != TSDB_CODE_SUCCESS) { + tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code)); + terrno = code; return -1; } @@ -128,8 +130,10 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("tmqsnap task start to execute"); - if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + int code = qExecTask(task, &pDataBlock, &ts); + if (code != 0) { + tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, tstrerror(code)); + terrno = code; return -1; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7d251fb074..a73deffa52 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,10 +180,10 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { - SExecTaskInfo* pTaskInfo = tinfo; - pTaskInfo->code = code; -} +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { +// SExecTaskInfo* pTaskInfo = tinfo; +// pTaskInfo->code = code; +//} int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { @@ -1080,7 +1080,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset) && pOffset->type != TMQ_OFFSET__SNAPSHOT_DATA) { + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } From 0b12224021345631b32ee889baddfaf13efcab28 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 May 2023 14:01:12 +0800 Subject: [PATCH 056/187] test: add schemless insert case --- .../system-test/eco-system/schemaless/insert.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/system-test/eco-system/schemaless/insert.py b/tests/system-test/eco-system/schemaless/insert.py index def70748d7..901196ebfd 100644 --- a/tests/system-test/eco-system/schemaless/insert.py +++ b/tests/system-test/eco-system/schemaless/insert.py @@ -104,34 +104,38 @@ class TDTestCase: tdLog.info(f" exe failed. i={i} {sql}") traceback.print_exc() + def genTags(self, i): + tags = f"t1={i},t2=abc,t3=work" + return tags # change table schema def schemaless_insert(self, change_cnt): # init ts = 1683194263000 for i in range(change_cnt): - t1 = i % 1000 index = int(i/10000) % 600 cols = self.genCols("c", 5, index, False) - tags = f"t1={t1},t2=abc,t3=work" - sql = f'sta,{tags} {cols} {ts + i}' + tags = self.genTags(index) + sql = f'{self.stable},{tags} {cols} {ts + i}' self.insert(sql, i) # run + def run(self): # seed #random.seed(int(time.time())) - self.dbname = "schema_change" + self.dbname = "eco_system" + self.stable = "sml_stb" # switch db tdSql.execute(f"use {self.dbname};") - tdSql.execute(f"drop table if exists sta;") + tdSql.execute(f"drop table if exists {self.stable};") # change meters try: - self.schemaless_insert(1000) + self.schemaless_insert(1000000) except: traceback.print_exc() From 8183be4afff416a66d9ec07879ac5e620d1e43d9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 May 2023 16:13:52 +0800 Subject: [PATCH 057/187] fix:move sleep logic from rpc thread to tmq thread & fix some error --- source/client/src/clientTmq.c | 9 ++++++--- source/dnode/vnode/src/tq/tq.c | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c08fbd0adf..b57ecfa845 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1275,7 +1275,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { // in case of consumer mismatch, wait for 500ms and retry if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { - taosMsleep(500); +// taosMsleep(500); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, wait for 500ms and set status to be RECOVER", tmq->consumerId); @@ -1289,8 +1289,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; taosWriteQitem(tmq->mqueue, pRspWrapper); - } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert - taosMsleep(500); +// } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert +// taosMsleep(5); } else{ tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, vgId, epoch, tstrerror(code), requestId); @@ -1731,6 +1731,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p // broadcast the poll request to all related vnodes static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { + if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){ + return 0; + } int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ce52277d71..91524f7a95 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -360,22 +360,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { // 2. check re-balance status if (pHandle->consumerId != consumerId) { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_INVALID_MSG; + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; taosWUnLockLatch(&pTq->lock); return -1; } // 3. update the epoch value - int32_t savedEpoch = pHandle->epoch; - if (savedEpoch <= reqEpoch) { - tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, - reqEpoch); - pHandle->epoch = reqEpoch; - }else { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", - consumerId, TD_VID(pTq->pVnode), req.subKey, savedEpoch, reqEpoch); + if (pHandle->epoch > reqEpoch) { + tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->epoch, reqEpoch); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; taosWUnLockLatch(&pTq->lock); return -1; @@ -395,6 +390,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } + if (pHandle->epoch < reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, pHandle->epoch, reqEpoch); + pHandle->epoch = reqEpoch; + } + char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, @@ -577,7 +577,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); } - atomic_add_fetch_32(&pHandle->epoch, 1); +// atomic_add_fetch_32(&pHandle->epoch, 1); // kill executing task if(tqIsHandleExec(pHandle)) { From 7b735c67d0a84f001c41c8f1145655cd3ddb40b4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 May 2023 16:36:25 +0800 Subject: [PATCH 058/187] fix:move sleep logic from rpc thread to tmq thread & fix some error --- source/dnode/vnode/src/tq/tq.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 91524f7a95..db03e97556 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -367,15 +367,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } - // 3. update the epoch value - if (pHandle->epoch > reqEpoch) { - tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", - consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->epoch, reqEpoch); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - taosWUnLockLatch(&pTq->lock); - return -1; - } - bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); @@ -390,6 +381,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } + // 3. update the epoch value if (pHandle->epoch < reqEpoch) { tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, pHandle->epoch, reqEpoch); pHandle->epoch = reqEpoch; From c9add944209c4e314ce54bf0f8975e4f8bdc7784 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 May 2023 09:58:52 +0800 Subject: [PATCH 059/187] docs: update readme main (#21401) * docs: update readme with libgflags * docs: update README-CN.md --- README-CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-CN.md b/README-CN.md index a6dfefc47a..cee6f97e9a 100644 --- a/README-CN.md +++ b/README-CN.md @@ -15,7 +15,7 @@ [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -简体中文 | [English](README.md) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/) +简体中文 | [English](README.md) | [TDengine 云服务](https://cloud.taosdata.com/?utm_medium=cn&utm_source=github) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/) # TDengine 简介 From 28d562a55d14d8e7dfb6b19e3081937fab3c239a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 22 May 2023 03:56:11 +0000 Subject: [PATCH 060/187] add parameter --- docs/en/14-reference/12-config/index.md | 19 +++++++++++++++++++ docs/zh/14-reference/12-config/index.md | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 430487a3af..cb9bebd723 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -97,6 +97,25 @@ The parameters described in this document by the effect that they have on the sy | Value Range | 10-50000000 | | Default Value | 5000 | +### numOfRpcSessions + +| Attribute | Description | +| ------------- | ---------------------------------------------------- | +| Applicable | Client/Server | +| Meaning | The maximum number of connection to create | +| Value Range | 100-100000 | +| Default Value | 10000 | + +### timeToGetAvailableConn + +| Attribute | Description | +| ------------- | ---------------------------------------------------- | +| Applicable | Client/Server | +| Meaning | The maximum waiting time to get avaliable conn | +| Value Range | 10-50000000(ms) | +| Default Value | 500000 | + + ## Monitoring Parameters :::note diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index e5efd77f80..9533f82ed8 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -97,6 +97,25 @@ taos --dump-config | 取值范围 | 10-50000000 | | 缺省值 | 5000 | +### numOfRpcSessions + +| 属性 | 说明 | +| -------- | ----------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 一个客户端能创建的最大连接数 | +| 取值范围 | 100-100000 | +| 缺省值 | 10000 | + +### timeToGetAvailableConn + +| 属性 | 说明 | +| -------- | ----------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 |获得可用连接的最长等待时间 | +| 取值范围 | 10-50000000(单位为毫秒) | +| 缺省值 | 500000 | + + ## 监控相关 :::note From 591b51f624139db99825be975199c25c7d094c2e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 May 2023 12:42:21 +0800 Subject: [PATCH 061/187] fix:set task code success after kill --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index db03e97556..f2956ea488 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -575,7 +575,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg if(tqIsHandleExec(pHandle)) { qTaskInfo_t pTaskInfo = pHandle->execHandle.task; if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); } // if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { From f67caaed1f5359ab806933603d40c41729e7ee24 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 22 May 2023 06:09:55 +0000 Subject: [PATCH 062/187] opt compile --- docs/zh/14-reference/12-config/index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 9533f82ed8..90070098b8 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -91,20 +91,20 @@ taos --dump-config ### maxShellConns | 属性 | 说明 | -| -------- | ----------------------- | +| --------| ----------------------- | | 适用范围 | 仅服务端适用 | -| 含义 | 一个 dnode 容许的连接数 | +| 含义 | 一个 dnode 容许的连接数 | | 取值范围 | 10-50000000 | -| 缺省值 | 5000 | +| 缺省值 | 5000 | ### numOfRpcSessions -| 属性 | 说明 | -| -------- | ----------------------- | -| 适用范围 | 客户端和服务端都适用 | -| 含义 | 一个客户端能创建的最大连接数 | +| 属性 | 说明 | +| --------| ---------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 一个客户端能创建的最大连接数| | 取值范围 | 100-100000 | -| 缺省值 | 10000 | +| 缺省值 | 10000 | ### timeToGetAvailableConn From d6b188f15d36a506a95ce3bf663700da9a61bc0c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 22 May 2023 06:10:00 +0000 Subject: [PATCH 063/187] opt compile --- docs/zh/14-reference/12-config/index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 90070098b8..ed6c9b4505 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -99,21 +99,21 @@ taos --dump-config ### numOfRpcSessions -| 属性 | 说明 | +| 属性 | 说明 | | --------| ---------------------- | | 适用范围 | 客户端和服务端都适用 | | 含义 | 一个客户端能创建的最大连接数| | 取值范围 | 100-100000 | -| 缺省值 | 10000 | +| 缺省值 | 10000 | ### timeToGetAvailableConn -| 属性 | 说明 | -| -------- | ----------------------- | -| 适用范围 | 客户端和服务端都适用 | -| 含义 |获得可用连接的最长等待时间 | -| 取值范围 | 10-50000000(单位为毫秒) | -| 缺省值 | 500000 | +| 属性 | 说明 | +| -------- | --------------------| +| 适用范围 | 客户端和服务端都适用 | +| 含义 |获得可用连接的最长等待时间| +| 取值范围 | 10-50000000(单位为毫秒)| +| 缺省值 | 500000 | ## 监控相关 From ae515851c4d753162f5bcd10aa940658e377055f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 May 2023 15:32:30 +0800 Subject: [PATCH 064/187] fix:remove get schema from previous for taosx --- source/dnode/vnode/src/meta/metaQuery.c | 38 ------------------------- 1 file changed, 38 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d464f64de3..428d201a17 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -387,42 +387,6 @@ _query: tDecoderClear(&dc); goto _exit; } - { // Traverse to find the previous qualified data - TBC *pCur; - tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); - STbDbKey key = {.version = sver, .uid = INT64_MAX}; - int c = 0; - tdbTbcMoveTo(pCur, &key, sizeof(key), &c); - if (c < 0) { - tdbTbcMoveToPrev(pCur); - } - - void *pKey = NULL; - void *pVal = NULL; - int vLen = 0, kLen = 0; - while (1) { - int32_t ret = tdbTbcPrev(pCur, &pKey, &kLen, &pVal, &vLen); - if (ret < 0) break; - - STbDbKey *tmp = (STbDbKey *)pKey; - if (tmp->uid != uid) { - continue; - } - SDecoder dcNew = {0}; - SMetaEntry meNew = {0}; - tDecoderInit(&dcNew, pVal, vLen); - metaDecodeEntry(&dcNew, &meNew); - pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow); - tDecoderClear(&dcNew); - tdbTbcClose(pCur); - tdbFree(pKey); - tdbFree(pVal); - goto _exit; - } - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCur); - } } else if (me.type == TSDB_CHILD_TABLE) { uid = me.ctbEntry.suid; tDecoderClear(&dc); @@ -447,7 +411,6 @@ _query: tDecoderClear(&dc); _exit: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } @@ -455,7 +418,6 @@ _exit: return pSchema; _err: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } From 7b20d2fa7d63af8959acaf29892a892aece67b71 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 22 May 2023 20:10:13 +0800 Subject: [PATCH 065/187] update crash_gen --- tests/pytest/crash_gen/crash_gen_main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index ec588659e9..9228cecdb5 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1722,12 +1722,14 @@ class TaskCreateDb(StateTransitionTask): vg_nums = random.randint(1, 8) cache_model = Dice.choice(['none', 'last_row', 'last_value', 'both']) buffer = random.randint(3, 128) + walRetentionPeriod = random.randint(1, 10000) dbName = self._db.getName() - self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} ".format(dbName, repStr, + self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} wal_retention_period {} ".format(dbName, repStr, updatePostfix, vg_nums, cache_model, - buffer)) + buffer, + walRetentionPeriod)) if dbName == "db_0" and Config.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix)) From d8387b95ff13682b3706bd9fbb06cf8094596b32 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 May 2023 20:14:43 +0800 Subject: [PATCH 066/187] fix:set task status to 0 in inti --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/executor/src/executor.c | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index f5f7218ab9..fe949d92ce 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,7 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); -//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f2956ea488..85d8d073f7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -370,7 +370,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); -// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); + qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a73deffa52..eea542e042 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,10 +180,10 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } -//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { -// SExecTaskInfo* pTaskInfo = tinfo; -// pTaskInfo->code = code; -//} +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { + SExecTaskInfo* pTaskInfo = tinfo; + pTaskInfo->code = code; +} int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { From ca479f57dfd40dc57503a33065f1798c7ff875fb Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Mon, 22 May 2023 21:40:38 +0800 Subject: [PATCH 067/187] Remove taosx, connector, explorer and taoskeeper in cloud tdengine --- packaging/docker/dockerbuild.sh | 10 ++++++++++ packaging/tools/makeclient.sh | 4 ++-- packaging/tools/makepkg.sh | 31 ++++++++++++++++++------------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 4b6fc8576b..ee09854176 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -123,6 +123,16 @@ else echo "Unknown cpuType: ${cpuType}" exit 1 fi +# check the tdengine cloud base image existed or not +if [ "$cloudBuild" == "y" ]; then + CloudBase=$(docker images | grep tdengine/tdengine-cloud-base ||:) + if [[ "$CloudBase" == "" ]]; then + echo "Rebuild tdengine cloud base image..." + docker build --rm -f "${communityDir}/packaging/docker/DockerfileCloud.base" -t tdengine/tdengine-cloud-base "." --build-arg cpuType=${cpuTypeAlias} + else + echo "Already found tdengine cloud base image" + fi +fi docker build --rm -f "${Dockerfile}" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias} if [ "$cloudBuild" != "y" ]; then diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 28dc770755..c4b074f903 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -175,7 +175,7 @@ if [ "$pagMode" == "lite" ]; then fi chmod a+x ${install_dir}/install_client.sh -if [[ $productName == "TDengine" ]]; then +if [[ $productName == "TDengine" ]] && [ "$verMode" != "cloud" ]; then # Copy example code mkdir -p ${install_dir}/examples examples_dir="${top_dir}/examples" @@ -191,7 +191,7 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json fi - if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then + if [ "$verMode" == "cluster" ]; then # Copy connector connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 3da005c405..7bf81d2807 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -69,25 +69,30 @@ if [ "$pagMode" == "lite" ]; then bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark " taostools_bin_files="" else - - wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ + if [ "$verMode" == "cloud" ]; then + taostools_bin_files=" ${build_dir}/bin/taosdump \ + ${build_dir}/bin/taosBenchmark" + else + wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ || echo "failed to download TDinsight.sh" - # download TDinsight caches - orig_pwd=$(pwd) - tdinsight_caches="" - cd ${build_dir}/bin/ && \ - chmod +x TDinsight.sh - ./TDinsight.sh --download-only ||: -# tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ") - cd $orig_pwd - echo "TDinsight caches: $tdinsight_caches" + # download TDinsight caches + orig_pwd=$(pwd) + tdinsight_caches="" + cd ${build_dir}/bin/ && \ + chmod +x TDinsight.sh + ./TDinsight.sh --download-only ||: + # tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ") + cd $orig_pwd + echo "TDinsight caches: $tdinsight_caches" - taostools_bin_files=" ${build_dir}/bin/taosdump \ + taostools_bin_files=" ${build_dir}/bin/taosdump \ ${build_dir}/bin/taosBenchmark \ ${build_dir}/bin/TDinsight.sh \ ${build_dir}/bin/tdengine-datasource.zip \ ${build_dir}/bin/tdengine-datasource.zip.md5sum" + fi + [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer') @@ -334,7 +339,7 @@ mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : # Copy connector -if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then +if [ "$verMode" == "cluster" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then From ed612088c05e40880520a1396f630131d51bd49a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 May 2023 10:32:13 +0800 Subject: [PATCH 068/187] fix:lost data because tsdbreader or task is killed --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 14 +++++++------- source/libs/executor/src/executor.c | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index fe949d92ce..f5f7218ab9 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,7 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 85d8d073f7..17eac7d096 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -370,7 +370,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); - qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); +// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; @@ -572,16 +572,16 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // atomic_add_fetch_32(&pHandle->epoch, 1); // kill executing task - if(tqIsHandleExec(pHandle)) { - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } +// if(tqIsHandleExec(pHandle)) { +// qTaskInfo_t pTaskInfo = pHandle->execHandle.task; +// if (pTaskInfo != NULL) { +// qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); +// } // if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { // qStreamCloseTsdbReader(pTaskInfo); // } - } +// } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); taosWUnLockLatch(&pTq->lock); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index eea542e042..a73deffa52 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,10 +180,10 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { - SExecTaskInfo* pTaskInfo = tinfo; - pTaskInfo->code = code; -} +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { +// SExecTaskInfo* pTaskInfo = tinfo; +// pTaskInfo->code = code; +//} int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { From 00524b3185f6eac4cec7148ccc4fd98d7cce5341 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Tue, 23 May 2023 11:36:42 +0800 Subject: [PATCH 069/187] remove the taosdump in cloud --- packaging/tools/makepkg.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 7bf81d2807..fc02a288ca 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -70,8 +70,7 @@ if [ "$pagMode" == "lite" ]; then taostools_bin_files="" else if [ "$verMode" == "cloud" ]; then - taostools_bin_files=" ${build_dir}/bin/taosdump \ - ${build_dir}/bin/taosBenchmark" + taostools_bin_files="${build_dir}/bin/taosBenchmark" else wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ From 13a6eb177c3c9682d2c269e0143d4065e4b732a6 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Tue, 23 May 2023 11:38:04 +0800 Subject: [PATCH 070/187] add space before the bin files --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index fc02a288ca..748b79bd7f 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -70,7 +70,7 @@ if [ "$pagMode" == "lite" ]; then taostools_bin_files="" else if [ "$verMode" == "cloud" ]; then - taostools_bin_files="${build_dir}/bin/taosBenchmark" + taostools_bin_files=" ${build_dir}/bin/taosBenchmark" else wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ From 8c1c2a2f976fd49657ea1ff7beb5642e3c86a1bb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 May 2023 17:47:17 +0800 Subject: [PATCH 071/187] fix:add grammar support --- source/libs/parser/inc/sql.y | 4 ++-- source/libs/parser/src/parAstCreater.c | 27 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 5918b488b4..4a457838d4 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -530,9 +530,9 @@ cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C). cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) WITH META AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, true); } cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, false); } + AS STABLE full_table_name(C) where_clause_opt(D). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, false, D); } cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - WITH META AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, true); } + WITH META AS STABLE full_table_name(C) where_clause_opt(D). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, true, D); } cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); } cmd ::= DROP CONSUMER GROUP exists_opt(A) cgroup_name(B) ON topic_name(C). { pCxt->pRootNode = createDropCGroupStmt(pCxt, A, &B, &C); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5a47ed731d..7258f6c415 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1713,6 +1713,33 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); pStmt->ignoreExists = ignoreExists; pStmt->withMeta = withMeta; + +// SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); +// if (NULL == pSelect) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// sprintf(pSelect->stmtName, "%p", pSelect); +// +// SRealTableNode* pRealTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); +// if (NULL == pRealTable) { +// nodesDestroyNode((SNode*)pSelect); +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb); +// snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); +// snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); +// pSelect->pFromTable = (SNode*)pRealTable; +// +// if (numOfProjs >= 0) { +// pSelect->pProjectionList = createProjectCols(numOfProjs, pProjCol); +// if (NULL == pSelect->pProjectionList) { +// nodesDestroyNode((SNode*)pSelect); +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// } +// +// pStmt->pQuery = pSelect; + strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName); strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pRealTable); From a54eaa20ffe37bcf6ffd59291ed0b52c530f78b5 Mon Sep 17 00:00:00 2001 From: xleili Date: Tue, 23 May 2023 18:11:00 +0800 Subject: [PATCH 072/187] build: release ver-3.0.4.2 --- cmake/cmake.version | 2 +- packaging/tools/makepkg.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 3166a0695c..1d43986de4 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.4.1") + SET(TD_VER_NUMBER "3.0.4.2") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 748b79bd7f..e297902bf9 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -345,7 +345,7 @@ if [ "$verMode" == "cluster" ]; then tmp_pwd=`pwd` cd ${install_dir}/connector if [ ! -d taos-connector-jdbc ];then - git clone -b 3.1.0 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + git clone -b 3.2.1 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: fi cd taos-connector-jdbc mvn clean package -Dmaven.test.skip=true From 34cf6aad3014516cb6bd69ea4797bc5a08295085 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 23 May 2023 18:29:53 +0800 Subject: [PATCH 073/187] fix: alter dnode option has no effect --- source/dnode/mnode/impl/src/mndDnode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 1f58ae97a3..e5ee4b8a21 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -965,7 +965,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); mInfo("dnode:%d, send config req to dnode, app:%p config:%s value:%s", cfgReq.dnodeId, pReq->info.ahandle, dcfgReq.config, dcfgReq.value); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .info = pReq->info}; + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; tmsgSendReq(&epSet, &rpcMsg); code = 0; } From 5bedf6b19ae337f77a43941ae6bc58e153d26517 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 23 May 2023 19:21:52 +0800 Subject: [PATCH 074/187] enh: support config batch rows number when import data from csv file --- include/common/tglobal.h | 2 +- source/common/src/tglobal.c | 10 +++---- source/libs/parser/src/parInsertSql.c | 4 ++- source/libs/parser/src/parInsertUtil.c | 37 +++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 6fde7b48a2..44276102cf 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -122,7 +122,7 @@ extern bool tsUseAdapter; // client extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; -extern int32_t tsMaxMemUsedByInsert; +extern int32_t tsMaxInsertBatchRows; // build info extern char version[]; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index b85dfa80b0..bef63d8a49 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -148,8 +148,8 @@ int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 database precision unit for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; -// maximum memory allowed to be allocated for a single csv load (in MB) -int32_t tsMaxMemUsedByInsert = 1024; +// maximum batch rows numbers imported from a single csv load +int32_t tsMaxInsertBatchRows = 1000000; float tsSelectivityRatio = 1.0; int32_t tsTagFilterResCacheSize = 1024 * 10; @@ -340,7 +340,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; - if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; + if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; @@ -725,7 +725,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { // tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; // tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; @@ -987,7 +987,7 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; } else if (strcasecmp("maxRetryWaitTime", name) == 0) { tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 02de9f227d..6751b34d8e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1552,7 +1552,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxInsertBatchRows) { pStmt->fileProcessing = true; break; } @@ -1561,6 +1561,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } taosMemoryFree(pLine); + parserDebug("0x%" PRIx64 " %d rows have been parsed", pCxt->pComCxt->requestId, *pNumOfRows); + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && !pStmt->fileProcessing) { code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index f921094752..952a927136 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -272,6 +272,41 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat return code; } +static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { + int32_t code = TSDB_CODE_SUCCESS; + SSubmitTbData* pTmp = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pTmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTmp->flags = pSrc->flags; + pTmp->suid = pSrc->suid; + pTmp->uid = pSrc->uid; + pTmp->sver = pSrc->sver; + pTmp->pCreateTbReq = pSrc->pCreateTbReq; + if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + pTmp->aCol = taosArrayInit(128, sizeof(SColData)); + if (NULL == pTmp->aCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } else { + pTmp->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pTmp->aRowP) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } + } + + taosMemoryFree(pSrc); + if (TSDB_CODE_SUCCESS == code) { + *pDst = pTmp; + } + + return code; +} + + static void resetColValues(SArray* pValues) { int32_t num = taosArrayGetSize(pValues); for (int32_t i = 0; i < num; ++i) { @@ -381,7 +416,7 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx } } taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); - taosMemoryFreeClear(pTableCxt->pData); + rebuildTableData(pTableCxt->pData, &pTableCxt->pData); qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); From b52f867a1f933772765b44d6b9a4297cc489a7fc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 24 May 2023 09:56:59 +0800 Subject: [PATCH 075/187] fix: rebuild table data issue --- source/libs/parser/src/parInsertUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 952a927136..de7d154db6 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -282,7 +282,7 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { pTmp->suid = pSrc->suid; pTmp->uid = pSrc->uid; pTmp->sver = pSrc->sver; - pTmp->pCreateTbReq = pSrc->pCreateTbReq; + pTmp->pCreateTbReq = NULL; if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { pTmp->aCol = taosArrayInit(128, sizeof(SColData)); if (NULL == pTmp->aCol) { From 9a421a92f90aefb83c54f870fe18c672a3bd904e Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 24 May 2023 10:32:10 +0800 Subject: [PATCH 076/187] fix: core dump when select with interval caused by order --- source/libs/executor/src/filloperator.c | 2 +- source/libs/executor/src/tfill.c | 2 +- tests/script/tsim/query/interval.sim | 36 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 73222ee375..6d4b8570d1 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -258,7 +258,7 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t STimeWindow w = {0}; int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - getInitialStartTimeWindow(pInterval, startKey, &w, order); + getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index c98746c241..a39521f25c 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -561,7 +561,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma ASSERT(numOfRes >= numOfRows); } else { // reach the end of data if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || - (ekey1 >= pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { + (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 833da4a8ba..e2b0d219cb 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -170,6 +170,42 @@ endi # return -1 #endi +print ================ step10 +print -------- create database and table +sql create database if not exists test +sql use test +sql create stable st (ts timestamp, c2 int) tags(tg int) +print -------- insert 300 rows data +$i = 0 +while $i < 300 + $t = 1577807983000 + $cc = $i * 1000 + $t = $t + $cc + sql select $i % 3 + if $data00 != 0.000000000 then + goto continue_while + endi + sql select $i % 4 + if $data00 == 0.000000000 then + goto continue_while + endi + sql insert into t1 using st tags(1) values ( $t , $i ) + continue_while: + $i = $i + 1 +endw + +$ms1 = 1577808120000 +$ms2 = 1577808000000 +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts asc +print ----> select asc rows: $rows +$asc_rows = $rows +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts desc +print ----> select desc rows: $rows +$desc_rows = $rows +if $desc_rows != $asc_rows then + return -1 +endi + print =============== clear #sql drop database $db #sql select * from information_schema.ins_databases From d6d0bf0e41ef5a0490376d2ce8658826868b06f9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 May 2023 10:47:26 +0800 Subject: [PATCH 077/187] enh: add varchar sma --- source/common/src/tdataformat.c | 52 ++++++++++++++++--- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index b18bd882ae..55204045ba 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2441,7 +2441,7 @@ _exit: int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, char *data) { int32_t code = 0; - if(data == NULL){ + if (data == NULL) { for (int32_t i = 0; i < nRows; ++i) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); } @@ -2455,8 +2455,9 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; } else { - if(ASSERT(varDataTLen(data + offset) <= bytes)){ - uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes); + if (ASSERT(varDataTLen(data + offset) <= bytes)) { + uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), + bytes); code = TSDB_CODE_INVALID_PARA; goto _exit; } @@ -2508,7 +2509,7 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { ASSERT(pColData->type == pBind->buffer_type); } - + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { @@ -3521,6 +3522,43 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum } } +static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = 0; + *numOfNull = 0; + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + case (HAS_NONE | HAS_NULL): + *numOfNull = pColData->nVal; + break; + case HAS_VALUE: + *numOfNull = 0; + break; + case (HAS_VALUE | HAS_NULL): + case (HAS_VALUE | HAS_NONE): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT1(pColData->pBitMap, iVal) == 0) { + (*numOfNull)++; + } + } + break; + case (HAS_VALUE | HAS_NONE | HAS_NULL): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT2(pColData->pBitMap, iVal) != 2) { + (*numOfNull)++; + } + } + break; + default: + ASSERT(0); + break; + } +} + void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = { NULL, tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL @@ -3530,14 +3568,14 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_ tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARCHAR tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_NCHAR tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON NULL, // TSDB_DATA_TYPE_VARBINARY NULL, // TSDB_DATA_TYPE_DECIMAL NULL, // TSDB_DATA_TYPE_BLOB diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 50fd9d7aa7..b25ab393da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -523,7 +523,7 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type) || ((pColData->flag & HAS_VALUE) == 0)) continue; + if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue; SColumnDataAgg sma = {.colId = pColData->cid}; tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); From 7359719429b5403bbfd3c28a645fad72b534b356 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Wed, 24 May 2023 13:57:59 +0800 Subject: [PATCH 078/187] implement TD-24378 --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 748b79bd7f..9c141a4f37 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -428,7 +428,7 @@ if [ "$exitcode" != "0" ]; then exit $exitcode fi -if [ -n "${taostools_bin_files}" ]; then +if [ -n "${taostools_bin_files}" ] && [ "$verMode" != "cloud" ]; then wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh" if [ "$osType" != "Darwin" ]; then tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || : From 89bea7100f5cf92f7aa332b167c6d6a850ddd539 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 24 May 2023 00:31:05 -0700 Subject: [PATCH 079/187] fix: fix count var type error --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 +++++++++++--- source/libs/executor/src/scanoperator.c | 5 +++-- source/libs/function/src/builtinsimpl.c | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cb83a561d2..5a7bebebff 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -184,7 +184,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pT SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); void tsdbReaderClose(STsdbReader *pReader); int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4acc784aee..5d970ce6c3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4973,7 +4973,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { return code; } -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { +static bool doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { + bool hasNullSMA = false; // do fill all null column value SMA info int32_t i = 0, j = 0; int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); @@ -4993,6 +4994,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; size++; + hasNullSMA = true; } j += 1; } @@ -5003,12 +5005,15 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; + hasNullSMA = true; } j++; } + + return hasNullSMA; } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool *hasNullSMA) { SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; int32_t code = 0; @@ -5072,7 +5077,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } // do fill all null column value SMA info - doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg); + if (doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg)) { + *hasNullSMA = true; + return TSDB_CODE_SUCCESS; + } size_t size = taosArrayGetSize(pSup->pColAgg); int32_t i = 0, j = 0; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a001d99408..c1430c4ce0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -228,12 +228,13 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { bool allColumnsHaveAgg = true; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + bool hasNullSMA = false; + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg, &hasNullSMA); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - if (!allColumnsHaveAgg) { + if (!allColumnsHaveAgg || hasNullSMA) { return false; } return true; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4c019b3e71..cdecc8b8cc 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -500,7 +500,7 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) { */ SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) { + if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; } else { if (pInputCol->hasNull) { From bd45785dfdd4e88b815069eabd8c6eeebd6f3c78 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 24 May 2023 01:30:48 -0700 Subject: [PATCH 080/187] add test cases --- tests/system-test/2-query/count_null.py | 144 ++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 tests/system-test/2-query/count_null.py diff --git a/tests/system-test/2-query/count_null.py b/tests/system-test/2-query/count_null.py new file mode 100644 index 0000000000..6d2c8db8d6 --- /dev/null +++ b/tests/system-test/2-query/count_null.py @@ -0,0 +1,144 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def check_results(self): + tdSql.query(f"select count(*) from tb1") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c2) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c3) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c4) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c5) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c6) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c7) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb1") + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select count(*) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c2) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c3) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c4) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c5) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c6) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c7) from tb2") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb2") + tdSql.checkData(0, 0, 0) + + for i in range (3, 6): + tdSql.query(f"select count(*) from tb{i}") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c2) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c3) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c4) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c5) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c6) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c7) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c8) from tb{i}") + tdSql.checkData(0, 0, 10000) + + + def run(self): + dbname = 'db' + tbnames = ['tb1', 'tb2', 'tb3', 'tb4', 'tb5', 'tb6'] + num_rows = 20000 + num_tables = 6 + ts_base = 1685548800000 + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + for i in range (num_tables): + tdSql.execute( + f'''create table if not exists {dbname}.{tbnames[i]} + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) + + ''' + ) + + + tdLog.printNoPrefix("==========step2:insert data") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[0]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[1]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, null, null, null, null, null, null, null, 'binary', 'nchar')") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i < num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i >= num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + + tdLog.printNoPrefix("==========step3:check result in memory") + self.check_results(); + + tdLog.printNoPrefix("==========step3:check result from disk") + tdSql.execute(f"flush database db") + self.check_results(); + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 8537e4e80e0bb24a11cc911a2d36a53e4438ef15 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 24 May 2023 18:26:55 +0800 Subject: [PATCH 081/187] feat:add tag filter for stable subscribe --- include/common/tmsg.h | 3 ++ include/libs/executor/executor.h | 2 ++ source/common/src/tmsg.c | 15 ++++++++++ source/dnode/mnode/impl/src/mndScheduler.c | 2 ++ source/dnode/mnode/impl/src/mndTopic.c | 2 ++ source/dnode/vnode/src/inc/tq.h | 2 ++ source/dnode/vnode/src/tq/tq.c | 27 +++++++++++------ source/dnode/vnode/src/tq/tqMeta.c | 20 +++++++++---- source/dnode/vnode/src/tq/tqRead.c | 35 +++++----------------- source/libs/executor/src/executil.c | 16 +++++++++- source/libs/parser/src/parTranslater.c | 1 + 11 files changed, 82 insertions(+), 43 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1ea9714bf9..8fd09afb90 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1949,6 +1949,7 @@ typedef struct { char* ast; char subStbName[TSDB_TABLE_FNAME_LEN]; }; + char* subStbFilterAst; } SCMCreateTopicReq; int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); @@ -2758,6 +2759,7 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR tlen += taosEncodeString(buf, pReq->qmsg); } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { tlen += taosEncodeFixedI64(buf, pReq->suid); + tlen += taosEncodeString(buf, pReq->qmsg); } return tlen; } @@ -2773,6 +2775,7 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { buf = taosDecodeString(buf, &pReq->qmsg); } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { + buf = taosDecodeString(buf, &pReq->qmsg); buf = taosDecodeFixedI64(buf, &pReq->suid); } return (void*)buf; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index b7e6c42e3b..e3a75ecabc 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -82,6 +82,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); +int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* pTagCond, void* pTagIndexCond, SArray **tableList); + /** * set the task Id, usually used by message queue process * @param tinfo diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7102e556cc..1eb73ab842 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3841,6 +3841,10 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeI32(&encoder, strlen(pReq->sql)) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tEncodeI32(&encoder, strlen(pReq->subStbFilterAst)) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->subStbFilterAst) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3879,6 +3883,15 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tDecodeI32(&decoder, &astLen) < 0) return -1; + if (astLen > 0) { + pReq->subStbFilterAst = taosMemoryCalloc(1, astLen + 1); + if (pReq->subStbFilterAst == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->subStbFilterAst) < 0) return -1; + } + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -3889,6 +3902,8 @@ void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { taosMemoryFreeClear(pReq->sql); if (TOPIC_SUB_TYPE__COLUMN == pReq->subType) { taosMemoryFreeClear(pReq->ast); + }else if(TOPIC_SUB_TYPE__TABLE == pReq->subType) { + taosMemoryFreeClear(pReq->subStbFilterAst); } } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 734f624be0..0248a195db 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -591,6 +591,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + } else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ + pVgEp->qmsg = taosStrdup(pTopic->ast); } else { pVgEp->qmsg = taosStrdup(""); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index f6da370916..7d71aae3f4 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -462,6 +462,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.stbUid = pStb->uid; mndReleaseStb(pMnode, pStb); + topicObj.ast = taosStrdup(pCreate->ast); + topicObj.astLen = strlen(pCreate->ast) + 1; } /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/ /*topicObj.ast = NULL;*/ diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e431ca4a01..7895690f94 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -72,6 +72,8 @@ typedef struct { typedef struct { int64_t suid; + char* qmsg; // SubPlanToString + SNode* node; } STqExecTb; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 311d637be8..be3d6bc614 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -70,6 +70,8 @@ static void destroyTqHandle(void* data) { } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { walCloseReader(pData->pWalReader); tqCloseReader(pData->execHandle.pTqReader); + taosMemoryFreeClear(pData->execHandle.execTb.qmsg); + nodesDestroyNode(pData->execHandle.execTb.node); } if(pData->msg != NULL) { rpcFreeCont(pData->msg->pCont); @@ -470,7 +472,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle tqHandle = {0}; pHandle = &tqHandle; - uint64_t oldConsumerId = pHandle->consumerId; memcpy(pHandle->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN); pHandle->consumerId = req.newConsumerId; pHandle->epoch = -1; @@ -514,14 +515,22 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); pHandle->execHandle.execTb.suid = req.suid; + pHandle->execHandle.execTb.qmsg = req.qmsg; + req.qmsg = NULL; - SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); - vnodeGetCtbIdList(pVnode, req.suid, tbUidList); - tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pVnode->config.vgId, req.suid); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); + if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s", terrstr()); + return -1; } + + SArray* tbUidList = NULL; + ret = qGetTableList(req.suid, pVnode->pMeta, pVnode, pHandle->execHandle.execTb.node, NULL, &tbUidList); + if(ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, req.subKey, pHandle->consumerId); + taosArrayDestroy(tbUidList); + goto end; + } + tqDebug("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, req.suid); pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); @@ -532,8 +541,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); - tqDebug("try to persist handle %s consumer:0x%" PRIx64 " , old consumer:0x%" PRIx64, req.subKey, - pHandle->consumerId, oldConsumerId); + tqDebug("try to persist handle %s consumer:0x%" PRIx64, req.subKey, + pHandle->consumerId); ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index f3ecaa08f6..ee7af5b2bf 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -37,6 +37,7 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tEncodeI64(pEncoder, pHandle->execHandle.execTb.suid) < 0) return -1; + if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1; } tEndEncode(pEncoder); return pEncoder->pos; @@ -64,6 +65,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tDecodeI64(pDecoder, &pHandle->execHandle.execTb.suid) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1; } tEndDecode(pDecoder); return 0; @@ -336,13 +338,19 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); - vnodeGetCtbIdList(pTq->pVnode, handle.execHandle.execTb.suid, tbUidList); - tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); + if (nodesStringToNode(handle.execHandle.execTb.qmsg, &handle.execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s", terrstr()); + return -1; } + + SArray* tbUidList = NULL; + int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, handle.execHandle.execTb.node, NULL, &tbUidList); + if(ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle.subKey, handle.consumerId); + taosArrayDestroy(tbUidList); + goto end; + } + tqDebug("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid); handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 1fbdb25528..38f5307384 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1040,34 +1040,15 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { - SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); - SMetaReader mr = {0}; - metaReaderInit(&mr, pTq->pVnode->pMeta, 0); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) { - uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i); - - int32_t code = metaGetTableEntryByUidCache(&mr, *id); - if (code != TSDB_CODE_SUCCESS) { - tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); - continue; - } - - tDecoderClear(&mr.coder); - if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pTqHandle->execHandle.execTb.suid) { - tqDebug("table uid %" PRId64 " does not add to tq handle", *id); - continue; - } - - tqDebug("table uid %" PRId64 " add to tq handle", *id); - taosArrayPush(qa, id); + SArray* list = NULL; + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, pTqHandle->execHandle.execTb.node, NULL, &list); + if(ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); + taosArrayDestroy(list); + return ret; } - - metaReaderClear(&mr); - if (taosArrayGetSize(qa) > 0) { - tqReaderAddTbUidList(pTqHandle->execHandle.pTqReader, qa); - } - - taosArrayDestroy(qa); + tqReaderSetTbUidList(pTqHandle->execHandle.pTqReader, list); + taosArrayDestroy(list); } else { tqReaderRemoveTbUidList(pTqHandle->execHandle.pTqReader, tbUidList); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c51dc39b5b..1fb35ae271 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -935,6 +935,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; SScalarParam output = {0}; + SArray* pUidTagList = NULL; tagFilterAssist ctx = {0}; ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); @@ -954,7 +955,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; // int64_t stt = taosGetTimestampUs(); - SArray* pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo)); + pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo)); copyExistedUids(pUidTagList, pUidList); FilterCondType condType = checkTagCond(pTagCond); @@ -1121,6 +1122,19 @@ _end: return code; } +int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* pTagCond, void* pTagIndexCond, SArray **tableList){ + SScanPhysiNode node = {0}; + node.suid = suid; + node.uid = suid; + node.tableType = TSDB_SUPER_TABLE; + STableListInfo* pTableListInfo = tableListCreate(); + int code = getTableList(metaHandle, pVnode, &node, pTagCond, pTagIndexCond, pTableListInfo, "qGetTableList"); + *tableList = pTableListInfo->pTableList; + pTableListInfo->pTableList = NULL; + tableListDestroy(pTableListInfo); + return code; +} + size_t getTableTagsBufLen(const SNodeList* pGroups) { size_t keyLen = 0; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f4c86d4849..f90fe59908 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5764,6 +5764,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); tNameGetFullDbName(&name, pReq->subDbName); tNameExtractFullName(&name, pReq->subStbName); + code = nodesNodeToString(pStmt->pQuery, false, &pReq->subStbFilterAst, NULL); } else if ('\0' != pStmt->subDbName[0]) { pReq->subType = TOPIC_SUB_TYPE__DB; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->subDbName, strlen(pStmt->subDbName)); From 357e86b9945c6a2765da47ddad6f6d82708a06ad Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 24 May 2023 18:59:47 +0800 Subject: [PATCH 082/187] feat:add tag filter for stable subscribe --- include/common/tmsg.h | 7 ++----- source/common/src/tmsg.c | 27 +++++++------------------- source/libs/parser/src/parTranslater.c | 4 +++- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8fd09afb90..a7de03fbce 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1945,11 +1945,8 @@ typedef struct { int8_t withMeta; char* sql; char subDbName[TSDB_DB_FNAME_LEN]; - union { - char* ast; - char subStbName[TSDB_TABLE_FNAME_LEN]; - }; - char* subStbFilterAst; + char* ast; + char subStbName[TSDB_TABLE_FNAME_LEN]; } SCMCreateTopicReq; int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 1eb73ab842..cb4d0c9034 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3832,19 +3832,16 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeI8(&encoder, pReq->withMeta) < 0) return -1; if (tEncodeCStr(&encoder, pReq->subDbName) < 0) return -1; if (TOPIC_SUB_TYPE__DB == pReq->subType) { - } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tEncodeCStr(&encoder, pReq->subStbName) < 0) return -1; } else { + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tEncodeCStr(&encoder, pReq->subStbName) < 0) return -1; + } if (tEncodeI32(&encoder, strlen(pReq->ast)) < 0) return -1; if (tEncodeCStr(&encoder, pReq->ast) < 0) return -1; } if (tEncodeI32(&encoder, strlen(pReq->sql)) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; - if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tEncodeI32(&encoder, strlen(pReq->subStbFilterAst)) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->subStbFilterAst) < 0) return -1; - } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3866,9 +3863,10 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeI8(&decoder, &pReq->withMeta) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->subDbName) < 0) return -1; if (TOPIC_SUB_TYPE__DB == pReq->subType) { - } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tDecodeCStrTo(&decoder, pReq->subStbName) < 0) return -1; } else { + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tDecodeCStrTo(&decoder, pReq->subStbName) < 0) return -1; + } if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (astLen > 0) { pReq->ast = taosMemoryCalloc(1, astLen + 1); @@ -3883,15 +3881,6 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } - if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tDecodeI32(&decoder, &astLen) < 0) return -1; - if (astLen > 0) { - pReq->subStbFilterAst = taosMemoryCalloc(1, astLen + 1); - if (pReq->subStbFilterAst == NULL) return -1; - if (tDecodeCStrTo(&decoder, pReq->subStbFilterAst) < 0) return -1; - } - } - tEndDecode(&decoder); tDecoderClear(&decoder); @@ -3900,10 +3889,8 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { taosMemoryFreeClear(pReq->sql); - if (TOPIC_SUB_TYPE__COLUMN == pReq->subType) { + if (TOPIC_SUB_TYPE__DB != pReq->subType) { taosMemoryFreeClear(pReq->ast); - }else if(TOPIC_SUB_TYPE__TABLE == pReq->subType) { - taosMemoryFreeClear(pReq->subStbFilterAst); } } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f90fe59908..7c11443acb 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5764,7 +5764,9 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); tNameGetFullDbName(&name, pReq->subDbName); tNameExtractFullName(&name, pReq->subStbName); - code = nodesNodeToString(pStmt->pQuery, false, &pReq->subStbFilterAst, NULL); + if(pStmt->pQuery != NULL) { + code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); + } } else if ('\0' != pStmt->subDbName[0]) { pReq->subType = TOPIC_SUB_TYPE__DB; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->subDbName, strlen(pStmt->subDbName)); From f2b438622a17cd76d5600289074fa192d52779c0 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 24 May 2023 19:47:56 +0800 Subject: [PATCH 083/187] skip some errors --- tests/pytest/crash_gen/crash_gen_main.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 9228cecdb5..5024f1e2fe 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2043,18 +2043,19 @@ class TdSuperTable: for topic in current_topic_list: topic_list.append(topic) - consumer.subscribe(topic_list) - - # consumer with random work life - time_start = time.time() - while 1: - res = consumer.poll(1) - consumer.commit(res) - if time.time() - time_start > random.randint(5, 50): - break try: + consumer.subscribe(topic_list) + + # consumer with random work life + time_start = time.time() + while 1: + res = consumer.poll(1) + consumer.commit(res) + if time.time() - time_start > random.randint(5, 50): + break consumer.unsubscribe() - except TmqError as e: + consumer.close() + except TmqError as err: # topic deleted by other threads pass return From 4a4e3d92758625d74dcc5529c158aa6720a7dc22 Mon Sep 17 00:00:00 2001 From: "chao.feng" Date: Tue, 23 May 2023 18:15:47 +0800 Subject: [PATCH 084/187] update replica number in case and only left the scenario of 3 replica --- tests/parallel_test/cases.task | 2 + tests/system-test/2-query/ts_3398.py | 56 ++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 tests/system-test/2-query/ts_3398.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index cf967c9553..350f14b05d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -342,6 +342,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3398.py -N 3 -n 3 + ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 diff --git a/tests/system-test/2-query/ts_3398.py b/tests/system-test/2-query/ts_3398.py new file mode 100644 index 0000000000..54d5c91804 --- /dev/null +++ b/tests/system-test/2-query/ts_3398.py @@ -0,0 +1,56 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.sqlset import * +import datetime + + +class TDTestCase: + """This test case is used to verify the aliasName of Node structure is not truncated + when sum clause is more than 65 bits. + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), False) + + def run(self): + # test case for https://jira.taosdata.com:18080/browse/TS-3405: + # create db + ret = tdSql.execute("CREATE DATABASE IF NOT EXISTS statistics1 REPLICA {} DURATION 14400m KEEP 5256000m,5256000m,5256000m PRECISION 'ms' MINROWS 100 MAXROWS 4096 COMP 2;".format(self.replicaVar)) + tdSql.execute("use statistics1;") + + # create stable + ret = tdSql.execute("CREATE STABLE IF NOT EXISTS statistics1.`g`(`day` timestamp,`run_state` tinyint) TAGS(`vin` binary(32));") + ret = tdSql.execute("CREATE STABLE IF NOT EXISTS statistics1.`b`(`day` timestamp, `total_heart` int) TAGS(`vin` binary(32));") + ret = tdSql.execute("CREATE STABLE IF NOT EXISTS statistics1.`tg`(`day` timestamp,`lt_4177` int,`f30_4177` int, `f35_4177` int) TAGS(`vin` binary(32));") + + # insert the data to table + ret = tdSql.execute("insert into d1001 using statistics1.`g` tags('NJHYNBSAS0000061') values (%s, %d)" % ("'2023-05-01'", 99)) + ret = tdSql.execute("insert into d2001 using statistics1.`b` tags('NJHYNBSAS0000061') values (%s, %d)" % ("'2023-05-01'", 99)) + ret = tdSql.execute("insert into d3001 using statistics1.`tg` tags('NJHYNBSAS0000061') values (%s, %d, %d, %d)" % ("'2023-05-01'", 99, 99, 99)) + + # execute the sql statements + ret = tdSql.query("SELECT b.`day` `day`,sum(CASE WHEN tg.lt_4177 IS NULL THEN 0 ELSE tg.lt_4177 END \ + + CASE WHEN tg.f35_4177 IS NULL THEN 0 ELSE tg.f35_4177 END) / 3600 es0,sum(CASE WHEN tg.lt_4177 \ + IS NULL THEN 0 ELSE tg.lt_4177 END + CASE WHEN tg.f35_4177 IS NULL THEN 0 ELSE tg.f35_4177 \ + END + CASE WHEN tg.f30_4177 IS NULL THEN 0 ELSE tg.f30_4177 END) / 3600 es1 FROM \ + statistics1.b b,statistics1.tg tg,statistics1.g g WHERE b.`day` = tg.`day` AND g.`day` = b.`day` \ + AND b.vin = tg.vin AND b.vin = g.vin AND b.`day` BETWEEN '2023-05-01' AND '2023-05-05' \ + AND b.vin = 'NJHYNBSAS0000061' AND g.vin IS NOT NULL AND b.vin IS NOT NULL AND tg.vin IS NOT NULL \ + GROUP BY b.`day`;") + # check the result + if 0.055 in tdSql.queryResult[0] and 0.0825 in tdSql.queryResult[0]: + tdLog.info("query result is correct") + else: + tdLog.info("query result is wrong") + + def stop(self): + # clear the db + tdSql.execute("drop database if exists statistics1;") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 459c1e6c0cbafc55bc05f34846442b14bc891bb1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 25 May 2023 13:30:43 +0800 Subject: [PATCH 085/187] fix: fix timestamp conversion error due to daylight saving time --- source/common/src/ttime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index dbb31374de..0aeb420b67 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -93,13 +93,13 @@ int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 'T'); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 'T'); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 'T'); } } else { if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 0); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 0); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 0); } } } From a57da3bdf791e0d9912d4b84dfeaa66595c88f0c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 25 May 2023 14:13:34 +0800 Subject: [PATCH 086/187] fix: fix log not compressed when logKeepDays set to positive value --- source/util/src/tlog.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e3b1c2fa2f..c405fb6a25 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -210,15 +210,15 @@ static void taosKeepOldLog(char *oldName) { (void)taosRenameFile(oldName, fileName); - if (tsLogKeepDays < 0) { - char compressFileName[LOG_FILE_NAME_LEN + 20]; - snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); - if (taosCompressFile(fileName, compressFileName) == 0) { - (void)taosRemoveFile(fileName); - } + char compressFileName[LOG_FILE_NAME_LEN + 20]; + snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); + if (taosCompressFile(fileName, compressFileName) == 0) { + (void)taosRemoveFile(fileName); } - taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays)); + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } } static void *taosThreadToOpenNewFile(void *param) { From abca0495b113f901f19d848e110f646c3c0c7607 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Thu, 25 May 2023 18:54:22 +0800 Subject: [PATCH 087/187] enh(taosAdapter): change taosAdapter to main branch --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index c67918351d..13826a1a74 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 565ca21 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 2713f4f6d44eb0ed996995a43943dc66b913e024 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 25 May 2023 19:01:58 +0800 Subject: [PATCH 088/187] feat: support create topic as stable with conditions --- include/common/ttokendef.h | 1 + include/libs/nodes/cmdnodes.h | 1 + include/libs/qcom/query.h | 7 + source/common/src/tmsg.c | 8 +- source/libs/parser/inc/parAst.h | 2 +- source/libs/parser/inc/parUtil.h | 1 + source/libs/parser/src/parAstCreater.c | 40 +- source/libs/parser/src/parAstParser.c | 5 + source/libs/parser/src/parTranslater.c | 106 +- source/libs/parser/src/parUtil.c | 16 + source/libs/parser/src/sql.c | 1764 +++++++++---------- source/libs/parser/test/parInitialCTest.cpp | 9 + source/libs/qcom/src/queryUtil.c | 12 + tests/script/tsim/tmq/topic.sim | 10 + 14 files changed, 1059 insertions(+), 923 deletions(-) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 10b5328e6d..d1a8f6d0c6 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -347,6 +347,7 @@ #define TK_WAL 329 + #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 #define TK_NK_ILLEGAL 602 diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 2323d044ec..0c1946163a 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -358,6 +358,7 @@ typedef struct SCreateTopicStmt { bool ignoreExists; bool withMeta; SNode* pQuery; + SNode* pWhere; } SCreateTopicStmt; typedef struct SDropTopicStmt { diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 3841210076..5abb491d3e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -51,6 +51,12 @@ typedef enum { TARGET_TYPE_OTHER, } ETargetType; +typedef enum { + TCOL_TYPE_COLUMN = 1, + TCOL_TYPE_TAG, + TCOL_TYPE_NONE, +} ETableColumnType; + #define QUERY_POLICY_VNODE 1 #define QUERY_POLICY_HYBRID 2 #define QUERY_POLICY_QNODE 3 @@ -257,6 +263,7 @@ void destroyQueryExecRes(SExecResult* pRes); int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len); char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); +void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst); void freeVgInfo(SDBVgInfo* vgInfo); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index cb4d0c9034..bc826b38da 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3836,8 +3836,12 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { if (tEncodeCStr(&encoder, pReq->subStbName) < 0) return -1; } - if (tEncodeI32(&encoder, strlen(pReq->ast)) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->ast) < 0) return -1; + if (pReq->ast && strlen(pReq->ast) > 0) { + if (tEncodeI32(&encoder, strlen(pReq->ast)) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->ast) < 0) return -1; + } else { + if (tEncodeI32(&encoder, 0) < 0) return -1; + } } if (tEncodeI32(&encoder, strlen(pReq->sql)) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index e08b77e681..9cfb469e68 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -206,7 +206,7 @@ SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName, bool withMeta); SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, - bool withMeta); + bool withMeta, SNode* pWhere); SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName); SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, SToken* pTopicName); SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 850571eea1..9632ccf0fb 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -115,6 +115,7 @@ int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes); void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request); +SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 7258f6c415..5ebafaa9f9 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -822,16 +822,9 @@ SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill) { SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { CHECK_PARSER_STATUS(pCxt); - SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + SNode* select = createSelectStmtImpl(isDistinct, pProjectionList, pTable); CHECK_OUT_OF_MEM(select); - select->isDistinct = isDistinct; - select->pProjectionList = pProjectionList; - select->pFromTable = pTable; - sprintf(select->stmtName, "%p", select); - select->isTimeLineResult = true; - select->onlyHasKeepOrderFunc = true; - select->timeRange = TSWINDOW_INITIALIZER; - return (SNode*)select; + return select; } static void setSubquery(SNode* pStmt) { @@ -1703,7 +1696,7 @@ SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, ST } SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, - bool withMeta) { + bool withMeta, SNode* pWhere) { CHECK_PARSER_STATUS(pCxt); if (!checkTopicName(pCxt, pTopicName)) { return NULL; @@ -1713,32 +1706,7 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); pStmt->ignoreExists = ignoreExists; pStmt->withMeta = withMeta; - -// SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); -// if (NULL == pSelect) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// sprintf(pSelect->stmtName, "%p", pSelect); -// -// SRealTableNode* pRealTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); -// if (NULL == pRealTable) { -// nodesDestroyNode((SNode*)pSelect); -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb); -// snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); -// snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); -// pSelect->pFromTable = (SNode*)pRealTable; -// -// if (numOfProjs >= 0) { -// pSelect->pProjectionList = createProjectCols(numOfProjs, pProjCol); -// if (NULL == pSelect->pProjectionList) { -// nodesDestroyNode((SNode*)pSelect); -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// } -// -// pStmt->pQuery = pSelect; + pStmt->pWhere = pWhere; strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName); strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5db1f5dbdb..dea0c4bd50 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -355,6 +355,11 @@ static int32_t collectMetaKeyFromCreateTopic(SCollectMetaKeyCxt* pCxt, SCreateTo if (NULL != pStmt->pQuery) { return collectMetaKeyFromQuery(pCxt, pStmt->pQuery); } + if (NULL != pStmt->pWhere) { + int32_t code = collectMetaKeyFromRealTableImpl(pCxt, pStmt->subDbName, pStmt->subSTbName, + AUTH_TYPE_READ); + return code; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7c11443acb..6de1bd4b85 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -55,6 +55,13 @@ typedef struct STranslateContext { bool showRewrite; } STranslateContext; +typedef struct SBuildTopicContext { + bool colExists; + bool colNotFound; + STableMeta* pMeta; + SNodeList* pTags; +} SBuildTopicContext; + typedef struct SFullDatabaseName { char fullDbName[TSDB_DB_FNAME_LEN]; } SFullDatabaseName; @@ -5789,12 +5796,107 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS return code; } +static int32_t addTagList(SNodeList** ppList, SNode* pNode) { + if (NULL == *ppList) { + *ppList = nodesMakeList(); + } + + nodesListStrictAppend(*ppList, pNode); + + return TSDB_CODE_SUCCESS; +} + +static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) { + SBuildTopicContext* pCxt = (SBuildTopicContext*)pContext; + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + ETableColumnType type; + getColumnTypeFromMeta(pCxt->pMeta, ((SColumnNode*)pNode)->colName, &type); + if (type == TCOL_TYPE_COLUMN) { + pCxt->colExists = true; + return DEAL_RES_ERROR; + } else if (type == TCOL_TYPE_TAG) { + addTagList(&pCxt->pTags, nodesCloneNode(pNode)); + } else { + pCxt->colNotFound = true; + return DEAL_RES_ERROR; + } + } else if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + if (0 == strcasecmp(pFunc->functionName, "tbname")) { + addTagList(&pCxt->pTags, nodesCloneNode(pNode)); + } + } + + return DEAL_RES_CONTINUE; +} + +static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta, SNodeList** ppProjection) { + SBuildTopicContext colCxt = {.colExists = false, .colNotFound = false, .pMeta = pMeta, .pTags = NULL}; + nodesWalkExprPostOrder(pStmt->pWhere, checkColumnTagsInCond, &colCxt); + if (colCxt.colNotFound) { + nodesDestroyList(colCxt.pTags); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Invalid column name"); + } else if (colCxt.colExists) { + nodesDestroyList(colCxt.pTags); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Columns are forbidden in where clause"); + } + if (NULL == colCxt.pTags) { + for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) { + SSchema* tag = &pMeta->schema[pMeta->tableInfo.numOfColumns + i]; + SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + strcpy(col->colName, tag->name); + strcpy(col->node.aliasName, col->colName); + strcpy(col->node.userAlias, col->colName); + addTagList(&colCxt.pTags, (SNode*)col); + } + } + + *ppProjection = colCxt.pTags; + return TSDB_CODE_SUCCESS; +} + +static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SNode** pSelect) { + SParseContext* pParCxt = pCxt->pParseCxt; + SRequestConnInfo connInfo = {.pTrans = pParCxt->pTransporter, + .requestId = pParCxt->requestId, + .requestObjRefId = pParCxt->requestRid, + .mgmtEps = pParCxt->mgmtEpSet}; + SName name; + STableMeta* pMeta = NULL; + int32_t code = getTableMetaImpl(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta); + if (code) { + taosMemoryFree(pMeta); + return code; + } + if (TSDB_SUPER_TABLE != pMeta->tableType) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used"); + } + + SNodeList* pProjection = NULL; + code = checkCollectTopicTags(pCxt, pStmt, pMeta, &pProjection); + if (TSDB_CODE_SUCCESS == code) { + SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + strcpy(realTable->table.dbName, pStmt->subDbName); + strcpy(realTable->table.tableName, pStmt->subSTbName); + strcpy(realTable->table.tableAlias, pStmt->subSTbName); + *pSelect = createSelectStmtImpl(true, pProjection, (SNode*)realTable); + ((SSelectStmt*)*pSelect)->pWhere = nodesCloneNode(pStmt->pWhere); + code = translateQuery(pCxt, *pSelect); + } + + taosMemoryFree(pMeta); + return code; +} + static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) { - if (NULL == pStmt->pQuery) { + if (NULL == pStmt->pQuery && NULL == pStmt->pWhere) { return TSDB_CODE_SUCCESS; } - if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { + if (pStmt->pWhere) { + return buildQueryForTableTopic(pCxt, pStmt, &pStmt->pQuery); + } else if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (!pSelect->isDistinct && (NULL != pSelect->pFromTable && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable)) && diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 14da6f8aab..644d1cca05 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -666,6 +666,22 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog return code; } + +SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { + SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == select) { + return NULL; + } + select->isDistinct = isDistinct; + select->pProjectionList = pProjectionList; + select->pFromTable = pTable; + sprintf(select->stmtName, "%p", select); + select->isTimeLineResult = true; + select->onlyHasKeepOrderFunc = true; + select->timeRange = TSWINDOW_INITIALIZER; + return (SNode*)select; +} + static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj** pHash) { if (NULL == *pHash) { *pHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 2e473455e4..41a4327842 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -140,18 +140,18 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 762 +#define YYNSTATE 764 #define YYNRULE 583 #define YYNRULE_WITH_ACTION 583 #define YYNTOKEN 330 -#define YY_MAX_SHIFT 761 -#define YY_MIN_SHIFTREDUCE 1136 -#define YY_MAX_SHIFTREDUCE 1718 -#define YY_ERROR_ACTION 1719 -#define YY_ACCEPT_ACTION 1720 -#define YY_NO_ACTION 1721 -#define YY_MIN_REDUCE 1722 -#define YY_MAX_REDUCE 2304 +#define YY_MAX_SHIFT 763 +#define YY_MIN_SHIFTREDUCE 1138 +#define YY_MAX_SHIFTREDUCE 1720 +#define YY_ERROR_ACTION 1721 +#define YY_ACCEPT_ACTION 1722 +#define YY_NO_ACTION 1723 +#define YY_MIN_REDUCE 1724 +#define YY_MAX_REDUCE 2306 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -218,609 +218,609 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2904) +#define YY_ACTTAB_COUNT (2902) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2116, 2010, 1881, 396, 434, 168, 636, 1734, 433, 1989, - /* 10 */ 672, 162, 46, 44, 1646, 1723, 2008, 642, 374, 1894, - /* 20 */ 393, 506, 1495, 440, 39, 38, 1942, 503, 45, 43, - /* 30 */ 42, 41, 40, 1576, 1791, 1493, 123, 2134, 1520, 122, - /* 40 */ 121, 120, 119, 118, 117, 116, 115, 114, 2279, 2084, - /* 50 */ 593, 671, 593, 2275, 194, 2275, 39, 38, 167, 1571, - /* 60 */ 45, 43, 42, 41, 40, 19, 1833, 340, 2281, 186, - /* 70 */ 2281, 186, 1501, 2276, 619, 2276, 619, 45, 43, 42, - /* 80 */ 41, 40, 2115, 501, 654, 2151, 502, 1758, 169, 2117, - /* 90 */ 675, 2119, 2120, 670, 182, 665, 1520, 758, 36, 298, - /* 100 */ 15, 735, 734, 733, 732, 405, 1931, 731, 730, 144, - /* 110 */ 725, 724, 723, 722, 721, 720, 719, 157, 715, 714, - /* 120 */ 713, 404, 403, 710, 709, 708, 175, 174, 594, 2241, - /* 130 */ 509, 654, 1320, 502, 1758, 123, 1578, 1579, 122, 121, - /* 140 */ 120, 119, 118, 117, 116, 115, 114, 1311, 697, 696, - /* 150 */ 695, 1315, 694, 1317, 1318, 693, 690, 179, 1326, 687, - /* 160 */ 1328, 1329, 684, 681, 49, 640, 1551, 1561, 654, 2219, - /* 170 */ 655, 1892, 1577, 1580, 1720, 39, 38, 360, 1993, 45, - /* 180 */ 43, 42, 41, 40, 1409, 1410, 1496, 1523, 1494, 133, - /* 190 */ 655, 1892, 387, 62, 1715, 2216, 539, 39, 38, 280, - /* 200 */ 165, 45, 43, 42, 41, 40, 39, 38, 1894, 191, - /* 210 */ 45, 43, 42, 41, 40, 1499, 1500, 1868, 1550, 1553, - /* 220 */ 1554, 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, 1569, - /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 46, 44, 1722, 179, - /* 240 */ 62, 343, 93, 1518, 393, 408, 1495, 62, 49, 407, - /* 250 */ 471, 2116, 2280, 485, 352, 2275, 484, 1576, 1989, 1493, - /* 260 */ 1994, 672, 132, 131, 130, 129, 128, 127, 126, 125, - /* 270 */ 124, 2279, 454, 1522, 486, 2276, 2278, 456, 432, 396, - /* 280 */ 431, 707, 2280, 1571, 1275, 2275, 211, 165, 2134, 19, - /* 290 */ 504, 593, 1765, 1714, 2275, 1894, 1501, 1274, 1605, 1685, - /* 300 */ 2084, 2279, 671, 196, 2010, 2276, 2277, 430, 618, 2281, - /* 310 */ 186, 2275, 518, 2116, 2276, 619, 386, 621, 66, 2007, - /* 320 */ 642, 758, 361, 672, 15, 1767, 617, 186, 1264, 655, - /* 330 */ 1892, 2276, 619, 2115, 444, 257, 2151, 655, 1892, 110, - /* 340 */ 2117, 675, 2119, 2120, 670, 189, 665, 1521, 133, 143, - /* 350 */ 2134, 150, 2175, 2204, 1606, 544, 55, 389, 2200, 487, - /* 360 */ 1578, 1579, 2084, 482, 671, 1266, 476, 475, 474, 473, - /* 370 */ 470, 469, 468, 467, 466, 462, 461, 460, 459, 342, - /* 380 */ 451, 450, 449, 613, 446, 445, 359, 1650, 1520, 1521, - /* 390 */ 1551, 1561, 189, 1520, 545, 2115, 1577, 1580, 2151, 189, - /* 400 */ 189, 110, 2117, 675, 2119, 2120, 670, 707, 665, 1191, - /* 410 */ 1496, 1190, 1494, 2295, 62, 2204, 1262, 630, 140, 389, - /* 420 */ 2200, 279, 50, 608, 641, 630, 140, 1745, 35, 391, - /* 430 */ 1600, 1601, 1602, 1603, 1604, 1608, 1609, 1610, 1611, 1499, - /* 440 */ 1500, 1192, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, - /* 450 */ 1560, 667, 663, 1569, 1570, 1572, 1573, 1574, 1575, 2, - /* 460 */ 12, 46, 44, 1354, 1355, 1495, 402, 401, 62, 393, - /* 470 */ 1279, 1495, 557, 556, 555, 516, 2084, 2003, 1493, 547, - /* 480 */ 137, 551, 1576, 1278, 1493, 550, 2280, 2116, 32, 1502, - /* 490 */ 549, 554, 368, 367, 39, 38, 548, 672, 45, 43, - /* 500 */ 42, 41, 40, 1522, 1523, 1426, 1427, 2116, 1571, 614, - /* 510 */ 609, 602, 1674, 699, 19, 1501, 1935, 669, 185, 2212, - /* 520 */ 2213, 1501, 138, 2217, 2134, 632, 184, 2212, 2213, 398, - /* 530 */ 138, 2217, 1937, 1939, 2051, 1191, 2084, 1190, 671, 1552, - /* 540 */ 758, 1425, 1428, 1519, 2134, 489, 758, 39, 38, 15, - /* 550 */ 228, 45, 43, 42, 41, 40, 2084, 153, 671, 605, - /* 560 */ 604, 1672, 1673, 1675, 1676, 1677, 189, 1192, 166, 2115, - /* 570 */ 1744, 580, 2151, 318, 189, 170, 2117, 675, 2119, 2120, - /* 580 */ 670, 246, 665, 28, 1708, 1578, 1579, 316, 73, 2115, - /* 590 */ 12, 72, 2151, 2094, 562, 334, 2117, 675, 2119, 2120, - /* 600 */ 670, 668, 665, 656, 2169, 655, 1892, 2102, 107, 572, - /* 610 */ 209, 497, 495, 492, 54, 1551, 1561, 2098, 612, 2084, - /* 620 */ 189, 1577, 1580, 242, 438, 141, 620, 2296, 213, 1496, - /* 630 */ 478, 1494, 504, 1884, 1765, 1496, 1794, 1494, 165, 565, - /* 640 */ 1944, 1944, 1505, 641, 559, 2134, 1895, 358, 373, 241, - /* 650 */ 62, 279, 195, 2100, 376, 1942, 1942, 1501, 1499, 1500, - /* 660 */ 1169, 289, 290, 665, 1499, 1500, 288, 1550, 1553, 1554, - /* 670 */ 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, 1569, 1570, - /* 680 */ 1572, 1573, 1574, 1575, 2, 46, 44, 1581, 109, 70, - /* 690 */ 202, 201, 69, 393, 639, 1495, 2003, 2116, 611, 1171, - /* 700 */ 1974, 1174, 1175, 557, 556, 555, 1576, 633, 1493, 365, - /* 710 */ 547, 137, 551, 477, 641, 698, 550, 618, 655, 1892, - /* 720 */ 2275, 549, 554, 368, 367, 630, 140, 548, 81, 80, - /* 730 */ 437, 1643, 1571, 193, 2134, 617, 186, 439, 1938, 1939, - /* 740 */ 2276, 619, 717, 244, 1944, 1501, 2084, 243, 671, 655, - /* 750 */ 1892, 383, 14, 13, 341, 655, 1892, 423, 1877, 1942, - /* 760 */ 421, 417, 413, 410, 430, 650, 1743, 2003, 448, 1944, - /* 770 */ 758, 106, 2077, 47, 463, 366, 388, 364, 363, 2115, - /* 780 */ 541, 103, 2151, 2116, 1942, 110, 2117, 675, 2119, 2120, - /* 790 */ 670, 1944, 665, 672, 1869, 655, 1892, 183, 397, 2204, - /* 800 */ 101, 543, 189, 389, 2200, 542, 1942, 90, 347, 1578, - /* 810 */ 1579, 372, 87, 573, 464, 2084, 188, 2078, 593, 1735, - /* 820 */ 2134, 2275, 553, 552, 2230, 1885, 277, 2212, 629, 362, - /* 830 */ 134, 628, 2084, 2275, 671, 1742, 2281, 186, 1887, 1551, - /* 840 */ 1561, 2276, 619, 630, 140, 1577, 1580, 245, 617, 186, - /* 850 */ 655, 1892, 1619, 2276, 619, 458, 1586, 2219, 12, 1496, - /* 860 */ 10, 1494, 1520, 593, 457, 2115, 2275, 1741, 2151, 517, - /* 870 */ 1879, 111, 2117, 675, 2119, 2120, 670, 657, 665, 2176, - /* 880 */ 622, 2281, 186, 2215, 2084, 2204, 2276, 619, 1499, 1500, - /* 890 */ 2201, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, - /* 900 */ 667, 663, 1569, 1570, 1572, 1573, 1574, 1575, 2, 46, - /* 910 */ 44, 659, 1523, 2176, 1468, 1469, 2084, 393, 427, 1495, - /* 920 */ 52, 2116, 3, 705, 155, 154, 702, 701, 700, 152, - /* 930 */ 1576, 633, 1493, 39, 38, 655, 1892, 45, 43, 42, - /* 940 */ 41, 40, 429, 425, 187, 2212, 2213, 2116, 138, 2217, - /* 950 */ 198, 1875, 655, 1892, 1889, 578, 1571, 672, 2134, 2238, - /* 960 */ 705, 155, 154, 702, 701, 700, 152, 399, 1520, 1501, - /* 970 */ 2084, 247, 671, 655, 1892, 165, 42, 41, 40, 2219, - /* 980 */ 1642, 39, 38, 1894, 2134, 45, 43, 42, 41, 40, - /* 990 */ 84, 543, 589, 83, 758, 542, 2084, 47, 671, 655, - /* 1000 */ 1892, 593, 1740, 2115, 2275, 2214, 2151, 2116, 1552, 110, - /* 1010 */ 2117, 675, 2119, 2120, 670, 2067, 665, 672, 634, 2281, - /* 1020 */ 186, 183, 9, 2204, 2276, 619, 1739, 389, 2200, 2115, - /* 1030 */ 1867, 87, 2151, 1578, 1579, 110, 2117, 675, 2119, 2120, - /* 1040 */ 670, 2094, 665, 254, 2134, 1738, 571, 2295, 2231, 2204, - /* 1050 */ 146, 2084, 135, 389, 2200, 1883, 2084, 1888, 671, 569, - /* 1060 */ 256, 567, 1989, 1551, 1561, 2098, 34, 655, 1892, 1577, - /* 1070 */ 1580, 1607, 39, 38, 1662, 2084, 45, 43, 42, 41, - /* 1080 */ 40, 729, 727, 1496, 1896, 1494, 638, 655, 1892, 2115, - /* 1090 */ 655, 1892, 2151, 164, 2084, 169, 2117, 675, 2119, 2120, - /* 1100 */ 670, 2100, 665, 1174, 1175, 312, 293, 200, 1921, 652, - /* 1110 */ 623, 665, 1499, 1500, 1552, 1550, 1553, 1554, 1555, 1556, - /* 1120 */ 1557, 1558, 1559, 1560, 667, 663, 1569, 1570, 1572, 1573, - /* 1130 */ 1574, 1575, 2, 46, 44, 1737, 2242, 2116, 655, 1892, - /* 1140 */ 1736, 393, 1733, 1495, 2094, 33, 1732, 672, 1731, 2251, - /* 1150 */ 655, 1892, 655, 1892, 1576, 1612, 1493, 653, 2102, 705, - /* 1160 */ 155, 154, 702, 701, 700, 152, 1730, 142, 2098, 299, - /* 1170 */ 2175, 400, 2224, 1639, 2134, 1944, 703, 1177, 704, 1935, - /* 1180 */ 1571, 1935, 1729, 1519, 2084, 2094, 2084, 1728, 671, 2084, - /* 1190 */ 1943, 2084, 2094, 1501, 1870, 2084, 625, 2084, 1727, 2102, - /* 1200 */ 1726, 1725, 2070, 251, 2100, 377, 2103, 718, 234, 2098, - /* 1210 */ 1854, 232, 74, 592, 665, 2084, 2098, 236, 758, 2115, - /* 1220 */ 235, 15, 2151, 2116, 422, 110, 2117, 675, 2119, 2120, - /* 1230 */ 670, 2084, 665, 672, 1639, 600, 2084, 2295, 148, 2204, - /* 1240 */ 153, 441, 546, 389, 2200, 2100, 390, 2084, 1781, 2084, - /* 1250 */ 2084, 415, 2100, 238, 442, 665, 237, 1578, 1579, 1774, - /* 1260 */ 2134, 82, 665, 240, 1260, 153, 239, 1772, 711, 575, - /* 1270 */ 558, 574, 2084, 662, 671, 64, 255, 64, 261, 1717, - /* 1280 */ 1718, 560, 14, 13, 666, 2105, 1834, 1551, 1561, 563, - /* 1290 */ 1240, 2244, 274, 1577, 1580, 227, 268, 1463, 606, 1832, - /* 1300 */ 1504, 1503, 712, 1831, 153, 2115, 48, 1496, 2151, 1494, - /* 1310 */ 2135, 110, 2117, 675, 2119, 2120, 670, 286, 665, 91, - /* 1320 */ 71, 151, 1466, 2295, 1238, 2204, 406, 64, 48, 389, - /* 1330 */ 2200, 1998, 1671, 53, 1670, 263, 1499, 1500, 2107, 1550, - /* 1340 */ 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, - /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 2116, 402, 401, - /* 1360 */ 1221, 637, 153, 1423, 48, 679, 1768, 672, 1509, 2269, - /* 1370 */ 151, 1764, 153, 1759, 291, 1932, 2234, 647, 295, 1576, - /* 1380 */ 136, 1502, 631, 2116, 1613, 1562, 276, 151, 273, 1, - /* 1390 */ 5, 409, 356, 672, 2134, 2223, 414, 1222, 1446, 306, - /* 1400 */ 199, 443, 1523, 1999, 480, 1571, 2084, 447, 671, 452, - /* 1410 */ 1518, 490, 465, 1991, 479, 472, 753, 491, 1501, 1305, - /* 1420 */ 2134, 311, 1332, 1597, 481, 488, 203, 1336, 626, 1343, - /* 1430 */ 2116, 204, 2084, 493, 671, 494, 206, 1341, 496, 2115, - /* 1440 */ 672, 498, 2151, 661, 156, 110, 2117, 675, 2119, 2120, - /* 1450 */ 670, 2116, 665, 1507, 1506, 1524, 4, 2295, 499, 2204, - /* 1460 */ 507, 672, 1526, 389, 2200, 2115, 500, 2134, 2151, 508, - /* 1470 */ 510, 110, 2117, 675, 2119, 2120, 670, 1521, 665, 2084, - /* 1480 */ 1525, 671, 214, 2295, 511, 2204, 512, 216, 2134, 389, - /* 1490 */ 2200, 1527, 513, 1194, 219, 515, 221, 85, 86, 519, - /* 1500 */ 2084, 536, 671, 225, 538, 537, 112, 346, 540, 1882, - /* 1510 */ 231, 1878, 2115, 2060, 577, 2151, 2057, 2056, 170, 2117, - /* 1520 */ 675, 2119, 2120, 670, 579, 665, 233, 89, 149, 307, - /* 1530 */ 158, 159, 1510, 2115, 1505, 248, 2151, 1880, 1876, 110, - /* 1540 */ 2117, 675, 2119, 2120, 670, 160, 665, 161, 583, 252, - /* 1550 */ 582, 2179, 1453, 2204, 584, 2116, 597, 389, 2200, 607, - /* 1560 */ 587, 1513, 1515, 250, 2250, 672, 590, 645, 259, 588, - /* 1570 */ 2297, 2235, 2245, 603, 663, 1569, 1570, 1572, 1573, 1574, - /* 1580 */ 1575, 262, 2249, 8, 616, 378, 610, 2226, 598, 596, - /* 1590 */ 595, 267, 2134, 379, 139, 272, 1639, 2298, 627, 1522, - /* 1600 */ 624, 2220, 382, 635, 2084, 269, 671, 1528, 2004, 308, - /* 1610 */ 643, 281, 96, 309, 644, 648, 98, 2018, 2017, 2016, - /* 1620 */ 649, 2116, 173, 385, 270, 1893, 271, 100, 61, 102, - /* 1630 */ 2185, 672, 1936, 313, 1855, 310, 754, 2115, 302, 755, - /* 1640 */ 2151, 757, 348, 110, 2117, 675, 2119, 2120, 670, 2116, - /* 1650 */ 665, 2274, 677, 275, 337, 2177, 51, 2204, 2134, 672, - /* 1660 */ 322, 389, 2200, 336, 326, 317, 2076, 2075, 349, 2074, - /* 1670 */ 2084, 315, 671, 78, 2071, 411, 412, 1486, 1487, 192, - /* 1680 */ 416, 2116, 2069, 418, 419, 420, 2134, 2068, 357, 2066, - /* 1690 */ 424, 672, 426, 2064, 428, 79, 1449, 1448, 2084, 2065, - /* 1700 */ 671, 2030, 2029, 2115, 2028, 435, 2151, 436, 2027, 110, - /* 1710 */ 2117, 675, 2119, 2120, 670, 2026, 665, 1982, 2134, 1400, - /* 1720 */ 1981, 658, 1979, 2204, 1978, 145, 1977, 389, 2200, 1980, - /* 1730 */ 2084, 2115, 671, 1976, 2151, 1975, 1973, 111, 2117, 675, - /* 1740 */ 2119, 2120, 670, 1972, 665, 1971, 197, 453, 1970, 455, - /* 1750 */ 1984, 2204, 2116, 1969, 1968, 2203, 2200, 483, 147, 1954, - /* 1760 */ 1953, 1952, 672, 2115, 1967, 1966, 2151, 1965, 1964, 111, - /* 1770 */ 2117, 675, 2119, 2120, 670, 1963, 665, 1962, 1961, 1960, - /* 1780 */ 2116, 1959, 1958, 2204, 1957, 1956, 1955, 660, 2200, 2134, - /* 1790 */ 672, 1983, 1951, 1950, 1402, 1949, 1948, 1947, 1946, 2116, - /* 1800 */ 1945, 2084, 1276, 671, 344, 1280, 345, 1797, 205, 669, - /* 1810 */ 1796, 1272, 1795, 1793, 207, 2116, 1754, 2134, 180, 1176, - /* 1820 */ 2104, 208, 1753, 2047, 2037, 672, 76, 2025, 77, 2084, - /* 1830 */ 210, 671, 2024, 218, 673, 220, 2134, 2151, 2002, 1871, - /* 1840 */ 111, 2117, 675, 2119, 2120, 670, 181, 665, 2084, 505, - /* 1850 */ 671, 212, 2134, 1792, 2204, 1790, 1214, 384, 351, 2200, - /* 1860 */ 520, 1788, 2115, 521, 2084, 2151, 671, 522, 328, 2117, - /* 1870 */ 675, 2119, 2120, 670, 526, 665, 2116, 524, 525, 1786, - /* 1880 */ 528, 2115, 529, 530, 2151, 1784, 672, 334, 2117, 675, - /* 1890 */ 2119, 2120, 670, 532, 665, 533, 2170, 2115, 581, 2116, - /* 1900 */ 2151, 534, 1771, 335, 2117, 675, 2119, 2120, 670, 672, - /* 1910 */ 665, 615, 1770, 2134, 1750, 1873, 761, 63, 392, 1348, - /* 1920 */ 1347, 1872, 230, 1263, 1261, 2084, 1259, 671, 1258, 1257, - /* 1930 */ 305, 1256, 1250, 1782, 1255, 369, 2134, 561, 1252, 726, - /* 1940 */ 1251, 394, 1249, 1775, 728, 370, 178, 1773, 2084, 371, - /* 1950 */ 671, 564, 751, 747, 743, 739, 303, 566, 2115, 1749, - /* 1960 */ 1748, 2151, 1747, 568, 335, 2117, 675, 2119, 2120, 670, - /* 1970 */ 570, 665, 113, 1473, 1475, 1472, 1477, 27, 2046, 1455, - /* 1980 */ 67, 2115, 2036, 1457, 2151, 56, 585, 335, 2117, 675, - /* 1990 */ 2119, 2120, 670, 2116, 665, 163, 108, 2023, 586, 296, - /* 2000 */ 2021, 253, 1459, 672, 375, 2280, 29, 20, 17, 591, - /* 2010 */ 1687, 258, 59, 6, 7, 599, 601, 60, 265, 260, - /* 2020 */ 1669, 31, 266, 2105, 171, 2116, 264, 21, 65, 30, - /* 2030 */ 2134, 651, 1661, 1707, 92, 672, 1708, 22, 1702, 278, - /* 2040 */ 1701, 380, 2084, 2116, 671, 1706, 1705, 381, 176, 2022, - /* 2050 */ 1636, 1635, 58, 672, 2020, 2019, 2001, 18, 94, 95, - /* 2060 */ 284, 646, 2134, 2000, 103, 97, 283, 23, 57, 297, - /* 2070 */ 285, 282, 1667, 294, 2084, 576, 671, 24, 2151, 287, - /* 2080 */ 2134, 330, 2117, 675, 2119, 2120, 670, 292, 665, 68, - /* 2090 */ 2116, 249, 2084, 11, 671, 99, 1588, 1587, 13, 1511, - /* 2100 */ 672, 1543, 1598, 678, 177, 190, 395, 2115, 1566, 2154, - /* 2110 */ 2151, 682, 1325, 319, 2117, 675, 2119, 2120, 670, 664, - /* 2120 */ 665, 685, 1564, 37, 16, 2115, 676, 2134, 2151, 1563, - /* 2130 */ 25, 320, 2117, 675, 2119, 2120, 670, 2116, 665, 2084, - /* 2140 */ 1535, 671, 674, 26, 688, 1333, 680, 672, 1330, 1327, - /* 2150 */ 683, 686, 691, 1321, 1319, 300, 1324, 1323, 1322, 2116, - /* 2160 */ 689, 692, 104, 1342, 105, 75, 1338, 1212, 706, 672, - /* 2170 */ 1244, 1243, 2115, 301, 2134, 2151, 1242, 1241, 321, 2117, - /* 2180 */ 675, 2119, 2120, 670, 2116, 665, 2084, 1239, 671, 1237, - /* 2190 */ 1236, 1235, 1270, 716, 672, 1233, 2134, 1232, 1231, 1230, - /* 2200 */ 1229, 1228, 1227, 1267, 1265, 1224, 2116, 1223, 2084, 1220, - /* 2210 */ 671, 1219, 1218, 1217, 1789, 736, 672, 737, 738, 2115, - /* 2220 */ 1787, 2134, 2151, 740, 741, 327, 2117, 675, 2119, 2120, - /* 2230 */ 670, 742, 665, 2084, 1785, 671, 744, 746, 745, 1783, - /* 2240 */ 748, 2115, 749, 2134, 2151, 750, 1769, 331, 2117, 675, - /* 2250 */ 2119, 2120, 670, 752, 665, 2084, 1166, 671, 304, 756, - /* 2260 */ 1721, 1721, 1497, 314, 759, 760, 2115, 1746, 1721, 2151, - /* 2270 */ 2116, 1721, 323, 2117, 675, 2119, 2120, 670, 1721, 665, - /* 2280 */ 672, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 1721, - /* 2290 */ 2116, 2151, 1721, 1721, 332, 2117, 675, 2119, 2120, 670, - /* 2300 */ 672, 665, 1721, 1721, 1721, 2116, 1721, 2134, 1721, 1721, - /* 2310 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 2084, - /* 2320 */ 1721, 671, 1721, 1721, 1721, 1721, 1721, 2134, 1721, 1721, - /* 2330 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 2084, - /* 2340 */ 1721, 671, 2134, 1721, 1721, 1721, 1721, 672, 1721, 1721, - /* 2350 */ 1721, 1721, 2115, 1721, 2084, 2151, 671, 1721, 324, 2117, - /* 2360 */ 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, 1721, - /* 2370 */ 1721, 1721, 2115, 1721, 2134, 2151, 1721, 1721, 333, 2117, - /* 2380 */ 675, 2119, 2120, 670, 1721, 665, 2084, 2115, 671, 1721, - /* 2390 */ 2151, 1721, 1721, 325, 2117, 675, 2119, 2120, 670, 1721, - /* 2400 */ 665, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, - /* 2410 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 2115, - /* 2420 */ 1721, 1721, 2151, 1721, 2116, 338, 2117, 675, 2119, 2120, - /* 2430 */ 670, 1721, 665, 1721, 672, 1721, 1721, 1721, 1721, 1721, - /* 2440 */ 1721, 1721, 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2450 */ 1721, 1721, 2116, 1721, 2084, 1721, 671, 1721, 1721, 1721, - /* 2460 */ 1721, 2134, 672, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2470 */ 1721, 2116, 1721, 2084, 1721, 671, 1721, 1721, 1721, 1721, - /* 2480 */ 1721, 672, 1721, 1721, 1721, 1721, 1721, 2115, 1721, 2134, - /* 2490 */ 2151, 1721, 1721, 339, 2117, 675, 2119, 2120, 670, 1721, - /* 2500 */ 665, 2084, 1721, 671, 1721, 1721, 2115, 1721, 2134, 2151, - /* 2510 */ 1721, 1721, 2128, 2117, 675, 2119, 2120, 670, 2116, 665, - /* 2520 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 672, 1721, - /* 2530 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 2151, 1721, 1721, - /* 2540 */ 2127, 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, - /* 2550 */ 1721, 1721, 1721, 2115, 1721, 2134, 2151, 1721, 1721, 2126, - /* 2560 */ 2117, 675, 2119, 2120, 670, 1721, 665, 2084, 1721, 671, - /* 2570 */ 1721, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, - /* 2580 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 1721, - /* 2590 */ 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2600 */ 2115, 672, 1721, 2151, 1721, 1721, 353, 2117, 675, 2119, - /* 2610 */ 2120, 670, 2134, 665, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2620 */ 1721, 1721, 1721, 1721, 2084, 1721, 671, 1721, 2134, 1721, - /* 2630 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2640 */ 2084, 2116, 671, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2650 */ 1721, 672, 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, - /* 2660 */ 2151, 2116, 1721, 354, 2117, 675, 2119, 2120, 670, 1721, - /* 2670 */ 665, 672, 1721, 2115, 1721, 1721, 2151, 1721, 2134, 350, - /* 2680 */ 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, - /* 2690 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 2134, 1721, - /* 2700 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, - /* 2710 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 672, 1721, - /* 2720 */ 1721, 1721, 1721, 2115, 1721, 1721, 2151, 1721, 1721, 355, - /* 2730 */ 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, - /* 2740 */ 1721, 1721, 229, 673, 1721, 2134, 2151, 1721, 1721, 330, - /* 2750 */ 2117, 675, 2119, 2120, 670, 1721, 665, 2084, 172, 671, - /* 2760 */ 1721, 1721, 1721, 1721, 535, 531, 527, 523, 226, 1721, - /* 2770 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2780 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2790 */ 2115, 1721, 1721, 2151, 1721, 1721, 329, 2117, 675, 2119, - /* 2800 */ 2120, 670, 1721, 665, 1721, 1721, 1721, 1721, 88, 1721, - /* 2810 */ 1721, 224, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2820 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2830 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2840 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2850 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2860 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2870 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 223, 217, - /* 2880 */ 1721, 1721, 1721, 222, 1721, 514, 1721, 1721, 1721, 1721, - /* 2890 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2900 */ 1721, 1721, 1721, 215, + /* 0 */ 503, 2118, 2282, 504, 1760, 2277, 36, 300, 1883, 442, + /* 10 */ 66, 674, 46, 44, 1648, 45, 43, 42, 41, 40, + /* 20 */ 395, 2281, 1497, 39, 38, 2278, 2280, 45, 43, 42, + /* 30 */ 41, 40, 1976, 1578, 1793, 1495, 657, 1894, 2136, 39, + /* 40 */ 38, 632, 140, 45, 43, 42, 41, 40, 657, 1894, + /* 50 */ 2086, 1724, 673, 342, 2012, 191, 39, 38, 573, 1573, + /* 60 */ 45, 43, 42, 41, 40, 19, 388, 133, 520, 2009, + /* 70 */ 644, 571, 1503, 569, 541, 132, 131, 130, 129, 128, + /* 80 */ 127, 126, 125, 124, 2117, 168, 2153, 1736, 198, 170, + /* 90 */ 2119, 677, 2121, 2122, 672, 511, 667, 760, 504, 1760, + /* 100 */ 15, 737, 736, 735, 734, 407, 62, 733, 732, 144, + /* 110 */ 727, 726, 725, 724, 723, 722, 721, 157, 717, 716, + /* 120 */ 715, 406, 405, 712, 711, 710, 175, 174, 84, 404, + /* 130 */ 403, 83, 1687, 656, 709, 1722, 1580, 1581, 1946, 1511, + /* 140 */ 622, 2298, 278, 2214, 631, 375, 134, 630, 1522, 2277, + /* 150 */ 1578, 123, 1504, 1944, 122, 121, 120, 119, 118, 117, + /* 160 */ 116, 115, 114, 2012, 619, 186, 1553, 1563, 1946, 2278, + /* 170 */ 621, 2282, 1579, 1582, 2277, 360, 1573, 400, 2010, 644, + /* 180 */ 1939, 1941, 656, 1944, 398, 1522, 1498, 460, 1496, 1503, + /* 190 */ 2281, 87, 162, 1717, 2278, 2279, 459, 39, 38, 376, + /* 200 */ 1896, 45, 43, 42, 41, 40, 410, 1944, 364, 582, + /* 210 */ 1522, 409, 62, 49, 663, 1501, 1502, 1889, 1552, 1555, + /* 220 */ 1556, 1557, 1558, 1559, 1560, 1561, 1562, 669, 665, 1571, + /* 230 */ 1572, 1574, 1575, 1576, 1577, 2, 46, 44, 632, 140, + /* 240 */ 1171, 345, 1523, 1520, 395, 620, 1497, 480, 2277, 12, + /* 250 */ 473, 10, 595, 487, 354, 2277, 486, 1578, 189, 1495, + /* 260 */ 443, 508, 229, 619, 186, 62, 2096, 505, 2278, 621, + /* 270 */ 2283, 186, 456, 444, 488, 2278, 621, 458, 172, 1173, + /* 280 */ 2104, 1176, 1177, 1573, 537, 533, 529, 525, 226, 19, + /* 290 */ 2100, 1523, 1716, 559, 558, 557, 1503, 1193, 1607, 1192, + /* 300 */ 549, 137, 553, 1512, 1747, 1507, 552, 202, 201, 14, + /* 310 */ 13, 551, 556, 370, 369, 50, 642, 550, 2282, 404, + /* 320 */ 403, 760, 363, 615, 15, 182, 2102, 378, 88, 1194, + /* 330 */ 479, 224, 1515, 1517, 446, 1524, 667, 1933, 634, 184, + /* 340 */ 2214, 2215, 1504, 138, 2219, 665, 1571, 1572, 1574, 1575, + /* 350 */ 1576, 1577, 664, 2086, 1608, 1746, 1411, 1412, 1428, 1429, + /* 360 */ 1580, 1581, 179, 484, 189, 189, 478, 477, 476, 475, + /* 370 */ 472, 471, 470, 469, 468, 464, 463, 462, 461, 344, + /* 380 */ 453, 452, 451, 1996, 448, 447, 361, 657, 1894, 1277, + /* 390 */ 1553, 1563, 657, 1894, 1427, 1430, 1579, 1582, 223, 217, + /* 400 */ 2221, 610, 1276, 222, 2086, 516, 133, 1356, 1357, 1621, + /* 410 */ 1498, 55, 1496, 546, 39, 38, 1710, 189, 45, 43, + /* 420 */ 42, 41, 40, 215, 1796, 1871, 2218, 1676, 35, 393, + /* 430 */ 1602, 1603, 1604, 1605, 1606, 1610, 1611, 1612, 1613, 1501, + /* 440 */ 1502, 1745, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, + /* 450 */ 1562, 669, 665, 1571, 1572, 1574, 1575, 1576, 1577, 2, + /* 460 */ 12, 46, 44, 2107, 489, 167, 1652, 189, 1497, 395, + /* 470 */ 367, 1497, 1522, 1835, 607, 606, 1674, 1675, 1677, 1678, + /* 480 */ 1679, 1495, 1578, 280, 1495, 179, 2118, 616, 611, 604, + /* 490 */ 2086, 559, 558, 557, 643, 1507, 671, 1588, 549, 137, + /* 500 */ 553, 1725, 1599, 1522, 552, 362, 1995, 1664, 1573, 551, + /* 510 */ 556, 370, 369, 211, 19, 550, 2109, 506, 1503, 1767, + /* 520 */ 1870, 1503, 123, 2136, 2053, 122, 121, 120, 119, 118, + /* 530 */ 117, 116, 115, 114, 614, 2086, 368, 673, 366, 365, + /* 540 */ 429, 543, 1524, 760, 213, 518, 760, 2005, 506, 15, + /* 550 */ 1767, 657, 1894, 2118, 707, 155, 154, 704, 703, 702, + /* 560 */ 152, 2136, 545, 674, 431, 427, 544, 656, 166, 2117, + /* 570 */ 247, 2153, 246, 320, 336, 2119, 677, 2121, 2122, 672, + /* 580 */ 670, 667, 658, 2171, 709, 1580, 1581, 318, 73, 2281, + /* 590 */ 2136, 72, 39, 38, 1940, 1941, 45, 43, 42, 41, + /* 600 */ 40, 389, 2086, 1521, 673, 101, 42, 41, 40, 165, + /* 610 */ 209, 499, 497, 494, 613, 1553, 1563, 1896, 1554, 39, + /* 620 */ 38, 1579, 1582, 45, 43, 42, 41, 40, 28, 1266, + /* 630 */ 1887, 1525, 1498, 12, 1496, 1498, 2117, 1496, 2153, 657, + /* 640 */ 1894, 110, 2119, 677, 2121, 2122, 672, 49, 667, 1554, + /* 650 */ 62, 143, 436, 150, 2177, 2206, 700, 435, 440, 391, + /* 660 */ 2202, 1501, 1502, 1879, 1501, 1502, 1268, 1552, 1555, 1556, + /* 670 */ 1557, 1558, 1559, 1560, 1561, 1562, 669, 665, 1571, 1572, + /* 680 */ 1574, 1575, 1576, 1577, 2, 46, 44, 1583, 109, 1946, + /* 690 */ 280, 643, 1281, 395, 87, 1497, 385, 2118, 595, 142, + /* 700 */ 620, 2277, 2177, 2277, 1944, 1280, 1578, 635, 1495, 707, + /* 710 */ 155, 154, 704, 703, 702, 152, 2283, 186, 619, 186, + /* 720 */ 1890, 2278, 621, 2278, 621, 282, 555, 554, 81, 80, + /* 730 */ 439, 1503, 1573, 193, 2136, 2096, 1946, 1193, 659, 1192, + /* 740 */ 2178, 2096, 641, 390, 2005, 1503, 2086, 701, 673, 2104, + /* 750 */ 1937, 1944, 228, 1991, 343, 1885, 1946, 425, 2221, 2100, + /* 760 */ 423, 419, 415, 412, 432, 2100, 62, 491, 93, 1194, + /* 770 */ 760, 1945, 643, 47, 2221, 398, 291, 292, 1744, 258, + /* 780 */ 2117, 290, 2153, 165, 2217, 110, 2119, 677, 2121, 2122, + /* 790 */ 672, 1896, 667, 1869, 1223, 2102, 379, 183, 194, 2206, + /* 800 */ 2216, 2102, 189, 391, 2202, 667, 1743, 32, 1742, 1580, + /* 810 */ 1581, 667, 8, 39, 38, 1741, 188, 45, 43, 42, + /* 820 */ 41, 40, 34, 652, 2232, 2005, 545, 2086, 39, 38, + /* 830 */ 544, 1224, 45, 43, 42, 41, 40, 657, 1894, 1553, + /* 840 */ 1563, 657, 1894, 39, 38, 1579, 1582, 45, 43, 42, + /* 850 */ 41, 40, 1322, 731, 729, 2086, 519, 2086, 623, 1498, + /* 860 */ 441, 1496, 2226, 1641, 2086, 657, 1894, 1313, 699, 698, + /* 870 */ 697, 1317, 696, 1319, 1320, 695, 692, 1609, 1328, 689, + /* 880 */ 1330, 1331, 686, 683, 450, 2069, 632, 140, 1501, 1502, + /* 890 */ 1740, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, + /* 900 */ 669, 665, 1571, 1572, 1574, 1575, 1576, 1577, 2, 46, + /* 910 */ 44, 245, 107, 1176, 1177, 62, 638, 395, 189, 1497, + /* 920 */ 719, 2118, 707, 155, 154, 704, 703, 702, 152, 141, + /* 930 */ 1578, 635, 1495, 657, 1894, 657, 1894, 1886, 661, 2086, + /* 940 */ 2178, 657, 1894, 1179, 1881, 657, 1894, 2118, 1946, 1521, + /* 950 */ 1739, 33, 465, 1645, 466, 399, 1573, 674, 2136, 1769, + /* 960 */ 1891, 1614, 595, 1944, 255, 2277, 1525, 1877, 165, 1503, + /* 970 */ 2086, 1738, 673, 1735, 632, 140, 1897, 1734, 1470, 1471, + /* 980 */ 2283, 186, 1733, 627, 2136, 2278, 621, 185, 2214, 2215, + /* 990 */ 314, 138, 2219, 1923, 760, 1898, 2086, 47, 673, 2086, + /* 1000 */ 434, 2096, 433, 164, 2117, 1991, 2153, 2118, 1732, 110, + /* 1010 */ 2119, 677, 2121, 2122, 672, 2104, 667, 674, 1525, 1731, + /* 1020 */ 2086, 183, 2086, 2206, 1641, 2100, 2086, 391, 2202, 432, + /* 1030 */ 2117, 2086, 2153, 1580, 1581, 110, 2119, 677, 2121, 2122, + /* 1040 */ 672, 244, 667, 401, 2136, 243, 1522, 2297, 2233, 2206, + /* 1050 */ 196, 165, 1730, 391, 2202, 1729, 2086, 2086, 673, 1896, + /* 1060 */ 547, 2102, 392, 1553, 1563, 657, 1894, 189, 2086, 1579, + /* 1070 */ 1582, 667, 657, 1894, 624, 187, 2214, 2215, 705, 138, + /* 1080 */ 2219, 1937, 1264, 1498, 281, 1496, 1719, 1720, 657, 1894, + /* 1090 */ 2117, 640, 2153, 1728, 424, 169, 2119, 677, 2121, 2122, + /* 1100 */ 672, 2086, 667, 251, 2086, 90, 349, 295, 1991, 374, + /* 1110 */ 668, 575, 1501, 1502, 195, 1552, 1555, 1556, 1557, 1558, + /* 1120 */ 1559, 1560, 1561, 1562, 669, 665, 1571, 1572, 1574, 1575, + /* 1130 */ 1576, 1577, 2, 46, 44, 596, 2243, 2118, 2096, 657, + /* 1140 */ 1894, 395, 2086, 1497, 657, 1894, 2072, 674, 74, 2253, + /* 1150 */ 657, 1894, 2105, 200, 1578, 1727, 1495, 153, 654, 657, + /* 1160 */ 1894, 564, 2100, 655, 706, 1872, 257, 1937, 720, 301, + /* 1170 */ 52, 1856, 3, 146, 2136, 135, 574, 577, 402, 576, + /* 1180 */ 1573, 234, 236, 548, 232, 235, 2086, 1836, 673, 153, + /* 1190 */ 242, 1783, 1554, 1503, 153, 417, 64, 82, 2102, 2079, + /* 1200 */ 238, 2080, 1644, 237, 2086, 1262, 567, 240, 667, 148, + /* 1210 */ 239, 561, 1776, 560, 54, 628, 241, 1506, 760, 1774, + /* 1220 */ 2117, 15, 2153, 2118, 64, 110, 2119, 677, 2121, 2122, + /* 1230 */ 672, 580, 667, 674, 562, 602, 256, 2297, 594, 2206, + /* 1240 */ 262, 565, 153, 391, 2202, 595, 1465, 595, 2277, 1505, + /* 1250 */ 2277, 1468, 1770, 1673, 254, 1737, 70, 1580, 1581, 69, + /* 1260 */ 2136, 14, 13, 2283, 186, 2283, 186, 48, 2278, 621, + /* 1270 */ 2278, 621, 2086, 106, 673, 288, 71, 595, 151, 91, + /* 1280 */ 2277, 1672, 608, 103, 2246, 1834, 275, 1553, 1563, 227, + /* 1290 */ 269, 1833, 2137, 1579, 1582, 2283, 186, 264, 408, 639, + /* 1300 */ 2278, 621, 755, 153, 625, 64, 2117, 1498, 2153, 1496, + /* 1310 */ 2000, 110, 2119, 677, 2121, 2122, 672, 48, 667, 48, + /* 1320 */ 681, 151, 153, 2297, 1425, 2206, 1761, 1766, 1934, 391, + /* 1330 */ 2202, 633, 293, 649, 2236, 297, 1501, 1502, 277, 1552, + /* 1340 */ 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, 669, 665, + /* 1350 */ 1571, 1572, 1574, 1575, 1576, 1577, 2, 2118, 53, 136, + /* 1360 */ 1307, 274, 1615, 713, 151, 1, 714, 674, 9, 2271, + /* 1370 */ 1509, 411, 416, 358, 1564, 308, 313, 1334, 1338, 1345, + /* 1380 */ 199, 1448, 445, 2118, 1525, 1242, 2001, 482, 1240, 449, + /* 1390 */ 454, 1520, 481, 674, 2136, 2225, 467, 1993, 474, 483, + /* 1400 */ 492, 490, 1508, 493, 203, 204, 2086, 495, 673, 496, + /* 1410 */ 206, 498, 500, 1526, 501, 4, 1343, 509, 1528, 502, + /* 1420 */ 2136, 156, 510, 214, 512, 1523, 513, 1527, 514, 1529, + /* 1430 */ 515, 216, 2086, 517, 673, 219, 221, 85, 86, 521, + /* 1440 */ 2117, 225, 2153, 1196, 540, 110, 2119, 677, 2121, 2122, + /* 1450 */ 672, 538, 667, 539, 2062, 2118, 2059, 2297, 2058, 2206, + /* 1460 */ 542, 112, 1884, 391, 2202, 674, 2117, 2240, 2153, 231, + /* 1470 */ 1880, 110, 2119, 677, 2121, 2122, 672, 233, 667, 348, + /* 1480 */ 158, 2118, 159, 2297, 579, 2206, 1882, 1878, 160, 391, + /* 1490 */ 2202, 674, 2136, 161, 89, 248, 585, 584, 1455, 589, + /* 1500 */ 592, 252, 609, 2237, 2086, 586, 673, 149, 581, 309, + /* 1510 */ 250, 599, 2252, 590, 2118, 647, 2247, 605, 2136, 380, + /* 1520 */ 2251, 612, 591, 260, 674, 7, 2228, 263, 618, 268, + /* 1530 */ 2086, 173, 673, 600, 598, 597, 273, 2300, 2117, 626, + /* 1540 */ 2153, 381, 629, 110, 2119, 677, 2121, 2122, 672, 1641, + /* 1550 */ 667, 2136, 139, 1524, 384, 2297, 271, 2206, 2222, 637, + /* 1560 */ 1530, 391, 2202, 2086, 2117, 673, 2153, 2006, 636, 110, + /* 1570 */ 2119, 677, 2121, 2122, 672, 283, 667, 2276, 96, 312, + /* 1580 */ 270, 2181, 310, 2206, 645, 272, 646, 391, 2202, 2020, + /* 1590 */ 2019, 2118, 311, 2018, 650, 276, 98, 2117, 387, 2153, + /* 1600 */ 651, 674, 110, 2119, 677, 2121, 2122, 672, 61, 667, + /* 1610 */ 102, 1895, 2187, 100, 2179, 1938, 2206, 679, 304, 756, + /* 1620 */ 391, 2202, 2118, 757, 315, 759, 51, 339, 2136, 1857, + /* 1630 */ 319, 2078, 674, 350, 351, 324, 338, 317, 328, 2077, + /* 1640 */ 2086, 2076, 673, 78, 2073, 413, 414, 1488, 1489, 192, + /* 1650 */ 2118, 418, 2071, 420, 421, 422, 2070, 359, 2068, 2136, + /* 1660 */ 674, 426, 2067, 2066, 428, 430, 79, 1451, 1450, 2032, + /* 1670 */ 2031, 2086, 2030, 673, 2117, 437, 2153, 438, 2029, 110, + /* 1680 */ 2119, 677, 2121, 2122, 672, 2118, 667, 2136, 2028, 1984, + /* 1690 */ 1402, 660, 1983, 2206, 1981, 674, 145, 391, 2202, 2086, + /* 1700 */ 1980, 673, 1979, 1982, 1978, 2117, 1977, 2153, 1975, 1974, + /* 1710 */ 111, 2119, 677, 2121, 2122, 672, 1973, 667, 455, 197, + /* 1720 */ 1972, 457, 2136, 1986, 2206, 1971, 1970, 1969, 2205, 2202, + /* 1730 */ 1968, 1967, 1966, 2117, 2086, 2153, 673, 1965, 111, 2119, + /* 1740 */ 677, 2121, 2122, 672, 1964, 667, 1963, 1962, 1961, 1960, + /* 1750 */ 1959, 1958, 2206, 1957, 147, 1956, 662, 2202, 1955, 1954, + /* 1760 */ 1985, 1953, 1952, 1404, 1951, 1950, 1949, 485, 675, 346, + /* 1770 */ 2153, 1948, 1947, 111, 2119, 677, 2121, 2122, 672, 2118, + /* 1780 */ 667, 347, 1799, 205, 1798, 1278, 1797, 2206, 207, 674, + /* 1790 */ 1282, 353, 2202, 1795, 208, 1756, 210, 1178, 1755, 2118, + /* 1800 */ 180, 2049, 1274, 2039, 2106, 2027, 220, 2026, 2004, 674, + /* 1810 */ 2118, 1873, 76, 1794, 1792, 212, 2136, 181, 77, 218, + /* 1820 */ 674, 507, 522, 1790, 524, 526, 523, 527, 2086, 1788, + /* 1830 */ 673, 528, 530, 532, 1786, 534, 2136, 531, 535, 1773, + /* 1840 */ 1772, 536, 1216, 1752, 1875, 63, 230, 2136, 2086, 1349, + /* 1850 */ 673, 1350, 1874, 1265, 1263, 1261, 1260, 1259, 1252, 2086, + /* 1860 */ 1258, 673, 2117, 1257, 2153, 1254, 1784, 111, 2119, 677, + /* 1870 */ 2121, 2122, 672, 1253, 667, 371, 728, 730, 1777, 372, + /* 1880 */ 1251, 2206, 2117, 1775, 2153, 373, 2203, 169, 2119, 677, + /* 1890 */ 2121, 2122, 672, 2117, 667, 2153, 2118, 563, 330, 2119, + /* 1900 */ 677, 2121, 2122, 672, 566, 667, 674, 2118, 1751, 568, + /* 1910 */ 1750, 570, 1749, 572, 113, 1475, 1477, 674, 1479, 1474, + /* 1920 */ 2048, 1461, 2118, 1459, 27, 1457, 67, 2038, 2244, 163, + /* 1930 */ 2025, 587, 671, 2136, 2023, 2282, 20, 17, 1689, 601, + /* 1940 */ 59, 617, 29, 5, 2136, 2086, 6, 673, 259, 386, + /* 1950 */ 603, 253, 261, 267, 2107, 60, 2086, 1671, 673, 2136, + /* 1960 */ 171, 265, 30, 266, 56, 1663, 593, 588, 92, 31, + /* 1970 */ 21, 2086, 377, 673, 1709, 65, 1710, 22, 18, 2117, + /* 1980 */ 1704, 2153, 1703, 382, 170, 2119, 677, 2121, 2122, 672, + /* 1990 */ 2117, 667, 2153, 2118, 1708, 337, 2119, 677, 2121, 2122, + /* 2000 */ 672, 1707, 667, 674, 383, 2117, 1638, 2153, 2118, 279, + /* 2010 */ 336, 2119, 677, 2121, 2122, 672, 583, 667, 674, 2172, + /* 2020 */ 58, 176, 1637, 2024, 2022, 2021, 2003, 95, 94, 286, + /* 2030 */ 2136, 23, 2002, 97, 763, 394, 2299, 287, 1669, 289, + /* 2040 */ 294, 648, 2086, 68, 673, 2136, 296, 299, 307, 99, + /* 2050 */ 396, 103, 24, 1590, 1589, 13, 57, 2086, 1513, 673, + /* 2060 */ 2156, 666, 1600, 37, 178, 1568, 1566, 2118, 1565, 177, + /* 2070 */ 753, 749, 745, 741, 305, 16, 2117, 674, 2153, 25, + /* 2080 */ 190, 337, 2119, 677, 2121, 2122, 672, 11, 667, 1545, + /* 2090 */ 1537, 2117, 26, 2153, 678, 680, 337, 2119, 677, 2121, + /* 2100 */ 2122, 672, 2118, 667, 2136, 397, 676, 1335, 1332, 682, + /* 2110 */ 684, 685, 674, 1329, 108, 687, 2086, 298, 673, 688, + /* 2120 */ 690, 693, 1323, 1327, 1326, 1325, 302, 691, 1344, 1321, + /* 2130 */ 694, 104, 1324, 1340, 105, 1214, 708, 75, 1246, 2136, + /* 2140 */ 1245, 1244, 1243, 1241, 1239, 1238, 1272, 1237, 718, 653, + /* 2150 */ 578, 2086, 2153, 673, 1232, 332, 2119, 677, 2121, 2122, + /* 2160 */ 672, 303, 667, 1235, 2118, 1234, 1233, 1269, 1231, 1230, + /* 2170 */ 1229, 1267, 1226, 1225, 674, 1222, 1221, 1220, 1219, 1791, + /* 2180 */ 738, 740, 1789, 742, 285, 2117, 1787, 2153, 739, 284, + /* 2190 */ 321, 2119, 677, 2121, 2122, 672, 743, 667, 744, 746, + /* 2200 */ 747, 2136, 748, 1785, 750, 751, 752, 1771, 754, 249, + /* 2210 */ 1168, 1748, 306, 2086, 758, 673, 1499, 316, 761, 762, + /* 2220 */ 1723, 2118, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2230 */ 1723, 674, 2118, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2240 */ 1723, 1723, 674, 1723, 1723, 1723, 1723, 2117, 1723, 2153, + /* 2250 */ 1723, 2118, 322, 2119, 677, 2121, 2122, 672, 2136, 667, + /* 2260 */ 1723, 674, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2136, + /* 2270 */ 2086, 1723, 673, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2280 */ 1723, 2086, 1723, 673, 1723, 1723, 1723, 1723, 2136, 1723, + /* 2290 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2300 */ 2086, 1723, 673, 1723, 2117, 1723, 2153, 1723, 1723, 323, + /* 2310 */ 2119, 677, 2121, 2122, 672, 2117, 667, 2153, 2118, 1723, + /* 2320 */ 329, 2119, 677, 2121, 2122, 672, 1723, 667, 674, 1723, + /* 2330 */ 1723, 1723, 1723, 1723, 2117, 1723, 2153, 2118, 1723, 333, + /* 2340 */ 2119, 677, 2121, 2122, 672, 1723, 667, 674, 1723, 1723, + /* 2350 */ 1723, 1723, 1723, 2118, 1723, 2136, 1723, 1723, 1723, 1723, + /* 2360 */ 1723, 1723, 1723, 674, 1723, 1723, 1723, 2086, 2118, 673, + /* 2370 */ 1723, 1723, 1723, 1723, 2136, 1723, 1723, 1723, 674, 1723, + /* 2380 */ 1723, 1723, 1723, 1723, 2118, 1723, 2086, 1723, 673, 1723, + /* 2390 */ 2136, 1723, 1723, 1723, 674, 1723, 1723, 1723, 1723, 1723, + /* 2400 */ 1723, 2117, 2086, 2153, 673, 2136, 325, 2119, 677, 2121, + /* 2410 */ 2122, 672, 1723, 667, 1723, 1723, 1723, 2086, 1723, 673, + /* 2420 */ 2117, 2136, 2153, 1723, 1723, 334, 2119, 677, 2121, 2122, + /* 2430 */ 672, 1723, 667, 2086, 1723, 673, 2117, 1723, 2153, 1723, + /* 2440 */ 1723, 326, 2119, 677, 2121, 2122, 672, 1723, 667, 1723, + /* 2450 */ 1723, 2117, 2118, 2153, 1723, 1723, 335, 2119, 677, 2121, + /* 2460 */ 2122, 672, 674, 667, 1723, 1723, 1723, 2117, 1723, 2153, + /* 2470 */ 1723, 1723, 327, 2119, 677, 2121, 2122, 672, 1723, 667, + /* 2480 */ 1723, 1723, 1723, 2118, 1723, 1723, 1723, 1723, 1723, 2136, + /* 2490 */ 1723, 1723, 1723, 674, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2500 */ 1723, 2086, 1723, 673, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2510 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2118, 1723, + /* 2520 */ 2136, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 674, 1723, + /* 2530 */ 1723, 1723, 2086, 1723, 673, 2117, 1723, 2153, 1723, 1723, + /* 2540 */ 340, 2119, 677, 2121, 2122, 672, 1723, 667, 1723, 2118, + /* 2550 */ 1723, 1723, 1723, 1723, 1723, 2136, 1723, 1723, 1723, 674, + /* 2560 */ 1723, 1723, 1723, 1723, 1723, 1723, 2117, 2086, 2153, 673, + /* 2570 */ 1723, 341, 2119, 677, 2121, 2122, 672, 1723, 667, 1723, + /* 2580 */ 1723, 1723, 1723, 1723, 1723, 1723, 2136, 1723, 1723, 1723, + /* 2590 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2086, 1723, + /* 2600 */ 673, 2117, 1723, 2153, 1723, 1723, 2130, 2119, 677, 2121, + /* 2610 */ 2122, 672, 1723, 667, 1723, 2118, 1723, 1723, 1723, 1723, + /* 2620 */ 1723, 1723, 1723, 1723, 1723, 674, 1723, 1723, 1723, 1723, + /* 2630 */ 1723, 1723, 2117, 1723, 2153, 1723, 2118, 2129, 2119, 677, + /* 2640 */ 2121, 2122, 672, 1723, 667, 1723, 674, 1723, 1723, 1723, + /* 2650 */ 1723, 1723, 2136, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2660 */ 1723, 1723, 2118, 1723, 2086, 1723, 673, 1723, 1723, 1723, + /* 2670 */ 1723, 1723, 674, 2136, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2680 */ 1723, 2118, 1723, 1723, 1723, 2086, 1723, 673, 1723, 1723, + /* 2690 */ 1723, 674, 1723, 1723, 1723, 1723, 1723, 1723, 2117, 2136, + /* 2700 */ 2153, 1723, 1723, 2128, 2119, 677, 2121, 2122, 672, 1723, + /* 2710 */ 667, 2086, 2118, 673, 1723, 1723, 1723, 1723, 2136, 2117, + /* 2720 */ 1723, 2153, 674, 1723, 355, 2119, 677, 2121, 2122, 672, + /* 2730 */ 2086, 667, 673, 1723, 1723, 1723, 1723, 1723, 2118, 1723, + /* 2740 */ 1723, 1723, 1723, 1723, 1723, 2117, 1723, 2153, 674, 2136, + /* 2750 */ 356, 2119, 677, 2121, 2122, 672, 1723, 667, 1723, 1723, + /* 2760 */ 1723, 2086, 1723, 673, 2117, 1723, 2153, 1723, 1723, 352, + /* 2770 */ 2119, 677, 2121, 2122, 672, 2136, 667, 1723, 1723, 1723, + /* 2780 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2086, 1723, 673, + /* 2790 */ 1723, 1723, 1723, 1723, 1723, 2117, 1723, 2153, 1723, 1723, + /* 2800 */ 357, 2119, 677, 2121, 2122, 672, 2118, 667, 1723, 1723, + /* 2810 */ 1723, 1723, 1723, 1723, 1723, 1723, 674, 1723, 1723, 1723, + /* 2820 */ 1723, 675, 1723, 2153, 1723, 1723, 332, 2119, 677, 2121, + /* 2830 */ 2122, 672, 1723, 667, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2840 */ 1723, 1723, 1723, 2136, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2850 */ 1723, 1723, 1723, 1723, 1723, 2086, 1723, 673, 1723, 1723, + /* 2860 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2870 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2880 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2117, + /* 2890 */ 1723, 2153, 1723, 1723, 331, 2119, 677, 2121, 2122, 672, + /* 2900 */ 1723, 667, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 333, 384, 371, 362, 401, 332, 401, 334, 405, 343, - /* 10 */ 343, 370, 12, 13, 14, 0, 399, 400, 377, 378, - /* 20 */ 20, 14, 22, 342, 8, 9, 385, 20, 12, 13, - /* 30 */ 14, 15, 16, 33, 0, 35, 21, 370, 20, 24, - /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 3, 382, - /* 50 */ 447, 384, 447, 450, 388, 450, 8, 9, 351, 59, - /* 60 */ 12, 13, 14, 15, 16, 65, 359, 386, 465, 466, - /* 70 */ 465, 466, 72, 470, 471, 470, 471, 12, 13, 14, - /* 80 */ 15, 16, 415, 337, 20, 418, 340, 341, 421, 422, - /* 90 */ 423, 424, 425, 426, 369, 428, 20, 97, 436, 437, - /* 100 */ 100, 67, 68, 69, 70, 71, 381, 73, 74, 75, + /* 0 */ 337, 333, 447, 340, 341, 450, 436, 437, 371, 342, + /* 10 */ 4, 343, 12, 13, 14, 12, 13, 14, 15, 16, + /* 20 */ 20, 466, 22, 8, 9, 470, 471, 12, 13, 14, + /* 30 */ 15, 16, 0, 33, 0, 35, 342, 343, 370, 8, + /* 40 */ 9, 342, 343, 12, 13, 14, 15, 16, 342, 343, + /* 50 */ 382, 0, 384, 386, 384, 361, 8, 9, 21, 59, + /* 60 */ 12, 13, 14, 15, 16, 65, 396, 361, 64, 399, + /* 70 */ 400, 34, 72, 36, 368, 24, 25, 26, 27, 28, + /* 80 */ 29, 30, 31, 32, 416, 332, 418, 334, 59, 421, + /* 90 */ 422, 423, 424, 425, 426, 337, 428, 97, 340, 341, + /* 100 */ 100, 67, 68, 69, 70, 71, 100, 73, 74, 75, /* 110 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, - /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 461, 462, - /* 130 */ 337, 20, 97, 340, 341, 21, 136, 137, 24, 25, - /* 140 */ 26, 27, 28, 29, 30, 31, 32, 112, 113, 114, - /* 150 */ 115, 116, 117, 118, 119, 120, 121, 370, 123, 124, - /* 160 */ 125, 126, 127, 128, 100, 20, 166, 167, 20, 420, - /* 170 */ 342, 343, 172, 173, 330, 8, 9, 390, 391, 12, - /* 180 */ 13, 14, 15, 16, 166, 167, 186, 20, 188, 361, - /* 190 */ 342, 343, 362, 100, 178, 446, 368, 8, 9, 59, - /* 200 */ 370, 12, 13, 14, 15, 16, 8, 9, 378, 361, - /* 210 */ 12, 13, 14, 15, 16, 215, 216, 0, 218, 219, + /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 99, 12, + /* 130 */ 13, 102, 101, 20, 64, 330, 136, 137, 370, 22, + /* 140 */ 472, 473, 443, 444, 445, 377, 447, 448, 20, 450, + /* 150 */ 33, 21, 35, 385, 24, 25, 26, 27, 28, 29, + /* 160 */ 30, 31, 32, 384, 465, 466, 166, 167, 370, 470, + /* 170 */ 471, 447, 172, 173, 450, 377, 59, 380, 399, 400, + /* 180 */ 383, 384, 20, 385, 362, 20, 186, 155, 188, 72, + /* 190 */ 466, 350, 370, 178, 470, 471, 164, 8, 9, 377, + /* 200 */ 378, 12, 13, 14, 15, 16, 401, 385, 367, 111, + /* 210 */ 20, 406, 100, 100, 97, 215, 216, 376, 218, 219, /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 0, 370, - /* 240 */ 100, 18, 102, 20, 20, 401, 22, 100, 100, 405, - /* 250 */ 27, 333, 447, 30, 65, 450, 33, 33, 343, 35, - /* 260 */ 391, 343, 24, 25, 26, 27, 28, 29, 30, 31, - /* 270 */ 32, 466, 49, 20, 51, 470, 471, 54, 185, 362, - /* 280 */ 187, 64, 447, 59, 22, 450, 338, 370, 370, 65, - /* 290 */ 342, 447, 344, 277, 450, 378, 72, 35, 109, 101, - /* 300 */ 382, 466, 384, 388, 384, 470, 471, 214, 447, 465, - /* 310 */ 466, 450, 64, 333, 470, 471, 396, 272, 4, 399, - /* 320 */ 400, 97, 99, 343, 100, 345, 465, 466, 35, 342, - /* 330 */ 343, 470, 471, 415, 111, 168, 418, 342, 343, 421, - /* 340 */ 422, 423, 424, 425, 426, 252, 428, 20, 361, 431, - /* 350 */ 370, 433, 434, 435, 165, 368, 361, 439, 440, 97, - /* 360 */ 136, 137, 382, 140, 384, 72, 143, 144, 145, 146, + /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 342, 343, + /* 240 */ 4, 18, 20, 20, 20, 447, 22, 81, 450, 236, + /* 250 */ 27, 238, 447, 30, 65, 450, 33, 33, 252, 35, + /* 260 */ 22, 14, 33, 465, 466, 100, 358, 20, 470, 471, + /* 270 */ 465, 466, 49, 35, 51, 470, 471, 54, 49, 43, + /* 280 */ 372, 45, 46, 59, 55, 56, 57, 58, 59, 65, + /* 290 */ 382, 20, 277, 67, 68, 69, 72, 20, 109, 22, + /* 300 */ 74, 75, 76, 186, 333, 188, 80, 141, 142, 1, + /* 310 */ 2, 85, 86, 87, 88, 100, 20, 91, 3, 12, + /* 320 */ 13, 97, 99, 20, 100, 369, 418, 419, 99, 52, + /* 330 */ 164, 102, 215, 216, 111, 20, 428, 381, 442, 443, + /* 340 */ 444, 445, 35, 447, 448, 228, 229, 230, 231, 232, + /* 350 */ 233, 234, 65, 382, 165, 333, 166, 167, 136, 137, + /* 360 */ 136, 137, 370, 140, 252, 252, 143, 144, 145, 146, /* 370 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, - /* 380 */ 157, 158, 159, 20, 161, 162, 163, 14, 20, 20, - /* 390 */ 166, 167, 252, 20, 13, 415, 172, 173, 418, 252, - /* 400 */ 252, 421, 422, 423, 424, 425, 426, 64, 428, 20, - /* 410 */ 186, 22, 188, 433, 100, 435, 35, 342, 343, 439, - /* 420 */ 440, 168, 100, 171, 342, 342, 343, 333, 239, 240, + /* 380 */ 157, 158, 159, 391, 161, 162, 163, 342, 343, 22, + /* 390 */ 166, 167, 342, 343, 172, 173, 172, 173, 169, 170, + /* 400 */ 420, 171, 35, 174, 382, 176, 361, 136, 137, 101, + /* 410 */ 186, 361, 188, 368, 8, 9, 101, 252, 12, 13, + /* 420 */ 14, 15, 16, 194, 0, 0, 446, 215, 239, 240, /* 430 */ 241, 242, 243, 244, 245, 246, 247, 248, 249, 215, - /* 440 */ 216, 52, 218, 219, 220, 221, 222, 223, 224, 225, + /* 440 */ 216, 333, 218, 219, 220, 221, 222, 223, 224, 225, /* 450 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - /* 460 */ 236, 12, 13, 136, 137, 22, 12, 13, 100, 20, - /* 470 */ 22, 22, 67, 68, 69, 393, 382, 395, 35, 74, - /* 480 */ 75, 76, 33, 35, 35, 80, 3, 333, 2, 35, - /* 490 */ 85, 86, 87, 88, 8, 9, 91, 343, 12, 13, - /* 500 */ 14, 15, 16, 20, 20, 136, 137, 333, 59, 257, - /* 510 */ 258, 259, 215, 379, 65, 72, 382, 343, 443, 444, - /* 520 */ 445, 72, 447, 448, 370, 442, 443, 444, 445, 380, - /* 530 */ 447, 448, 383, 384, 366, 20, 382, 22, 384, 166, - /* 540 */ 97, 172, 173, 20, 370, 97, 97, 8, 9, 100, - /* 550 */ 35, 12, 13, 14, 15, 16, 382, 44, 384, 262, - /* 560 */ 263, 264, 265, 266, 267, 268, 252, 52, 18, 415, - /* 570 */ 333, 111, 418, 23, 252, 421, 422, 423, 424, 425, - /* 580 */ 426, 413, 428, 44, 101, 136, 137, 37, 38, 415, - /* 590 */ 236, 41, 418, 358, 4, 421, 422, 423, 424, 425, - /* 600 */ 426, 427, 428, 429, 430, 342, 343, 372, 348, 19, - /* 610 */ 60, 61, 62, 63, 101, 166, 167, 382, 343, 382, - /* 620 */ 252, 172, 173, 33, 361, 365, 472, 473, 338, 186, - /* 630 */ 81, 188, 342, 373, 344, 186, 0, 188, 370, 49, - /* 640 */ 370, 370, 188, 342, 54, 370, 378, 377, 377, 59, - /* 650 */ 100, 168, 168, 418, 419, 385, 385, 72, 215, 216, - /* 660 */ 4, 130, 131, 428, 215, 216, 135, 218, 219, 220, + /* 460 */ 236, 12, 13, 47, 97, 351, 14, 252, 22, 20, + /* 470 */ 37, 22, 20, 359, 262, 263, 264, 265, 266, 267, + /* 480 */ 268, 35, 33, 168, 35, 370, 333, 257, 258, 259, + /* 490 */ 382, 67, 68, 69, 342, 188, 343, 14, 74, 75, + /* 500 */ 76, 0, 215, 20, 80, 390, 391, 101, 59, 85, + /* 510 */ 86, 87, 88, 338, 65, 91, 100, 342, 72, 344, + /* 520 */ 0, 72, 21, 370, 366, 24, 25, 26, 27, 28, + /* 530 */ 29, 30, 31, 32, 343, 382, 103, 384, 105, 106, + /* 540 */ 181, 108, 20, 97, 338, 393, 97, 395, 342, 100, + /* 550 */ 344, 342, 343, 333, 129, 130, 131, 132, 133, 134, + /* 560 */ 135, 370, 129, 343, 205, 206, 133, 20, 18, 416, + /* 570 */ 361, 418, 414, 23, 421, 422, 423, 424, 425, 426, + /* 580 */ 427, 428, 429, 430, 64, 136, 137, 37, 38, 3, + /* 590 */ 370, 41, 8, 9, 383, 384, 12, 13, 14, 15, + /* 600 */ 16, 362, 382, 20, 384, 348, 14, 15, 16, 370, + /* 610 */ 60, 61, 62, 63, 423, 166, 167, 378, 166, 8, + /* 620 */ 9, 172, 173, 12, 13, 14, 15, 16, 44, 35, + /* 630 */ 373, 20, 186, 236, 188, 186, 416, 188, 418, 342, + /* 640 */ 343, 421, 422, 423, 424, 425, 426, 100, 428, 166, + /* 650 */ 100, 431, 401, 433, 434, 435, 111, 406, 361, 439, + /* 660 */ 440, 215, 216, 371, 215, 216, 72, 218, 219, 220, /* 670 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 99, - /* 690 */ 141, 142, 102, 20, 393, 22, 395, 333, 423, 43, - /* 700 */ 0, 45, 46, 67, 68, 69, 33, 343, 35, 37, - /* 710 */ 74, 75, 76, 164, 342, 111, 80, 447, 342, 343, - /* 720 */ 450, 85, 86, 87, 88, 342, 343, 91, 178, 179, - /* 730 */ 180, 4, 59, 183, 370, 465, 466, 361, 383, 384, - /* 740 */ 470, 471, 72, 131, 370, 72, 382, 135, 384, 342, - /* 750 */ 343, 377, 1, 2, 204, 342, 343, 207, 371, 385, - /* 760 */ 210, 211, 212, 213, 214, 393, 333, 395, 361, 370, - /* 770 */ 97, 100, 401, 100, 361, 103, 377, 105, 106, 415, - /* 780 */ 108, 110, 418, 333, 385, 421, 422, 423, 424, 425, - /* 790 */ 426, 370, 428, 343, 0, 342, 343, 433, 377, 435, - /* 800 */ 348, 129, 252, 439, 440, 133, 385, 195, 196, 136, - /* 810 */ 137, 199, 350, 201, 361, 382, 452, 401, 447, 334, - /* 820 */ 370, 450, 355, 356, 460, 373, 443, 444, 445, 367, - /* 830 */ 447, 448, 382, 450, 384, 333, 465, 466, 376, 166, - /* 840 */ 167, 470, 471, 342, 343, 172, 173, 130, 465, 466, - /* 850 */ 342, 343, 101, 470, 471, 155, 14, 420, 236, 186, - /* 860 */ 238, 188, 20, 447, 164, 415, 450, 333, 418, 361, - /* 870 */ 371, 421, 422, 423, 424, 425, 426, 432, 428, 434, - /* 880 */ 44, 465, 466, 446, 382, 435, 470, 471, 215, 216, - /* 890 */ 440, 218, 219, 220, 221, 222, 223, 224, 225, 226, + /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 370, + /* 690 */ 168, 342, 22, 20, 350, 22, 377, 333, 447, 431, + /* 700 */ 447, 450, 434, 450, 385, 35, 33, 343, 35, 129, + /* 710 */ 130, 131, 132, 133, 134, 135, 465, 466, 465, 466, + /* 720 */ 376, 470, 471, 470, 471, 59, 355, 356, 178, 179, + /* 730 */ 180, 72, 59, 183, 370, 358, 370, 20, 432, 22, + /* 740 */ 434, 358, 393, 377, 395, 72, 382, 379, 384, 372, + /* 750 */ 382, 385, 35, 343, 204, 372, 370, 207, 420, 382, + /* 760 */ 210, 211, 212, 213, 214, 382, 100, 97, 102, 52, + /* 770 */ 97, 385, 342, 100, 420, 362, 130, 131, 333, 168, + /* 780 */ 416, 135, 418, 370, 446, 421, 422, 423, 424, 425, + /* 790 */ 426, 378, 428, 0, 35, 418, 419, 433, 388, 435, + /* 800 */ 446, 418, 252, 439, 440, 428, 333, 2, 333, 136, + /* 810 */ 137, 428, 39, 8, 9, 333, 452, 12, 13, 14, + /* 820 */ 15, 16, 2, 393, 460, 395, 129, 382, 8, 9, + /* 830 */ 133, 72, 12, 13, 14, 15, 16, 342, 343, 166, + /* 840 */ 167, 342, 343, 8, 9, 172, 173, 12, 13, 14, + /* 850 */ 15, 16, 97, 355, 356, 382, 361, 382, 272, 186, + /* 860 */ 361, 188, 250, 251, 382, 342, 343, 112, 113, 114, + /* 870 */ 115, 116, 117, 118, 119, 120, 121, 165, 123, 124, + /* 880 */ 125, 126, 127, 128, 361, 0, 342, 343, 215, 216, + /* 890 */ 333, 218, 219, 220, 221, 222, 223, 224, 225, 226, /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 12, - /* 910 */ 13, 432, 20, 434, 197, 198, 382, 20, 181, 22, - /* 920 */ 42, 333, 44, 129, 130, 131, 132, 133, 134, 135, - /* 930 */ 33, 343, 35, 8, 9, 342, 343, 12, 13, 14, - /* 940 */ 15, 16, 205, 206, 443, 444, 445, 333, 447, 448, - /* 950 */ 59, 371, 342, 343, 361, 401, 59, 343, 370, 345, - /* 960 */ 129, 130, 131, 132, 133, 134, 135, 362, 20, 72, - /* 970 */ 382, 361, 384, 342, 343, 370, 14, 15, 16, 420, - /* 980 */ 253, 8, 9, 378, 370, 12, 13, 14, 15, 16, - /* 990 */ 99, 129, 361, 102, 97, 133, 382, 100, 384, 342, - /* 1000 */ 343, 447, 333, 415, 450, 446, 418, 333, 166, 421, - /* 1010 */ 422, 423, 424, 425, 426, 0, 428, 343, 361, 465, - /* 1020 */ 466, 433, 39, 435, 470, 471, 333, 439, 440, 415, - /* 1030 */ 0, 350, 418, 136, 137, 421, 422, 423, 424, 425, - /* 1040 */ 426, 358, 428, 406, 370, 333, 21, 433, 460, 435, - /* 1050 */ 42, 382, 44, 439, 440, 372, 382, 376, 384, 34, - /* 1060 */ 168, 36, 343, 166, 167, 382, 2, 342, 343, 172, - /* 1070 */ 173, 165, 8, 9, 101, 382, 12, 13, 14, 15, - /* 1080 */ 16, 355, 356, 186, 371, 188, 361, 342, 343, 415, - /* 1090 */ 342, 343, 418, 168, 382, 421, 422, 423, 424, 425, - /* 1100 */ 426, 418, 428, 45, 46, 363, 361, 388, 366, 361, - /* 1110 */ 274, 428, 215, 216, 166, 218, 219, 220, 221, 222, + /* 910 */ 13, 130, 348, 45, 46, 100, 401, 20, 252, 22, + /* 920 */ 72, 333, 129, 130, 131, 132, 133, 134, 135, 365, + /* 930 */ 33, 343, 35, 342, 343, 342, 343, 373, 432, 382, + /* 940 */ 434, 342, 343, 14, 371, 342, 343, 333, 370, 20, + /* 950 */ 333, 239, 361, 4, 361, 377, 59, 343, 370, 345, + /* 960 */ 361, 249, 447, 385, 361, 450, 20, 371, 370, 72, + /* 970 */ 382, 333, 384, 333, 342, 343, 378, 333, 197, 198, + /* 980 */ 465, 466, 333, 44, 370, 470, 471, 443, 444, 445, + /* 990 */ 363, 447, 448, 366, 97, 371, 382, 100, 384, 382, + /* 1000 */ 185, 358, 187, 168, 416, 343, 418, 333, 333, 421, + /* 1010 */ 422, 423, 424, 425, 426, 372, 428, 343, 20, 333, + /* 1020 */ 382, 433, 382, 435, 251, 382, 382, 439, 440, 214, + /* 1030 */ 416, 382, 418, 136, 137, 421, 422, 423, 424, 425, + /* 1040 */ 426, 131, 428, 362, 370, 135, 20, 433, 460, 435, + /* 1050 */ 388, 370, 333, 439, 440, 333, 382, 382, 384, 378, + /* 1060 */ 13, 418, 419, 166, 167, 342, 343, 252, 382, 172, + /* 1070 */ 173, 428, 342, 343, 44, 443, 444, 445, 379, 447, + /* 1080 */ 448, 382, 35, 186, 361, 188, 136, 137, 342, 343, + /* 1090 */ 416, 361, 418, 333, 209, 421, 422, 423, 424, 425, + /* 1100 */ 426, 382, 428, 371, 382, 195, 196, 361, 343, 199, + /* 1110 */ 371, 201, 215, 216, 168, 218, 219, 220, 221, 222, /* 1120 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, - /* 1130 */ 233, 234, 235, 12, 13, 333, 462, 333, 342, 343, - /* 1140 */ 333, 20, 333, 22, 358, 239, 333, 343, 333, 345, - /* 1150 */ 342, 343, 342, 343, 33, 249, 35, 361, 372, 129, - /* 1160 */ 130, 131, 132, 133, 134, 135, 333, 431, 382, 361, - /* 1170 */ 434, 361, 250, 251, 370, 370, 379, 14, 379, 382, - /* 1180 */ 59, 382, 333, 20, 382, 358, 382, 333, 384, 382, - /* 1190 */ 385, 382, 358, 72, 0, 382, 44, 382, 333, 372, - /* 1200 */ 333, 333, 0, 371, 418, 419, 372, 357, 104, 382, - /* 1210 */ 360, 107, 111, 48, 428, 382, 382, 104, 97, 415, - /* 1220 */ 107, 100, 418, 333, 209, 421, 422, 423, 424, 425, - /* 1230 */ 426, 382, 428, 343, 251, 345, 382, 433, 44, 435, - /* 1240 */ 44, 22, 13, 439, 440, 418, 419, 382, 0, 382, - /* 1250 */ 382, 49, 418, 104, 35, 428, 107, 136, 137, 0, - /* 1260 */ 370, 160, 428, 104, 35, 44, 107, 0, 13, 200, - /* 1270 */ 22, 202, 382, 65, 384, 44, 59, 44, 44, 136, - /* 1280 */ 137, 22, 1, 2, 371, 47, 359, 166, 167, 22, - /* 1290 */ 35, 392, 474, 172, 173, 346, 457, 101, 463, 358, - /* 1300 */ 35, 35, 13, 358, 44, 415, 44, 186, 418, 188, - /* 1310 */ 370, 421, 422, 423, 424, 425, 426, 44, 428, 102, - /* 1320 */ 44, 44, 101, 433, 35, 435, 346, 44, 44, 439, - /* 1330 */ 440, 392, 101, 168, 101, 101, 215, 216, 100, 218, + /* 1130 */ 233, 234, 235, 12, 13, 461, 462, 333, 358, 342, + /* 1140 */ 343, 20, 382, 22, 342, 343, 0, 343, 111, 345, + /* 1150 */ 342, 343, 372, 388, 33, 333, 35, 44, 361, 342, + /* 1160 */ 343, 4, 382, 361, 379, 0, 168, 382, 357, 361, + /* 1170 */ 42, 360, 44, 42, 370, 44, 19, 200, 361, 202, + /* 1180 */ 59, 104, 104, 13, 107, 107, 382, 359, 384, 44, + /* 1190 */ 33, 0, 166, 72, 44, 49, 44, 160, 418, 401, + /* 1200 */ 104, 401, 253, 107, 382, 35, 49, 104, 428, 44, + /* 1210 */ 107, 54, 0, 22, 101, 276, 59, 35, 97, 0, + /* 1220 */ 416, 100, 418, 333, 44, 421, 422, 423, 424, 425, + /* 1230 */ 426, 401, 428, 343, 22, 345, 59, 433, 48, 435, + /* 1240 */ 44, 22, 44, 439, 440, 447, 101, 447, 450, 35, + /* 1250 */ 450, 101, 0, 101, 407, 334, 99, 136, 137, 102, + /* 1260 */ 370, 1, 2, 465, 466, 465, 466, 44, 470, 471, + /* 1270 */ 470, 471, 382, 100, 384, 44, 44, 447, 44, 102, + /* 1280 */ 450, 101, 463, 110, 392, 358, 474, 166, 167, 346, + /* 1290 */ 457, 358, 370, 172, 173, 465, 466, 101, 346, 101, + /* 1300 */ 470, 471, 50, 44, 274, 44, 416, 186, 418, 188, + /* 1310 */ 392, 421, 422, 423, 424, 425, 426, 44, 428, 44, + /* 1320 */ 44, 44, 44, 433, 101, 435, 341, 343, 381, 439, + /* 1330 */ 440, 449, 101, 101, 392, 101, 215, 216, 467, 218, /* 1340 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 333, 12, 13, - /* 1360 */ 35, 101, 44, 101, 44, 44, 0, 343, 22, 345, - /* 1370 */ 44, 343, 44, 341, 101, 381, 392, 101, 101, 33, - /* 1380 */ 44, 35, 449, 333, 101, 101, 467, 44, 441, 451, - /* 1390 */ 254, 417, 416, 343, 370, 345, 49, 72, 184, 403, - /* 1400 */ 42, 389, 20, 392, 165, 59, 382, 389, 384, 387, - /* 1410 */ 20, 98, 342, 342, 387, 389, 50, 354, 72, 101, - /* 1420 */ 370, 101, 101, 215, 387, 96, 353, 101, 276, 101, - /* 1430 */ 333, 342, 382, 95, 384, 352, 342, 101, 342, 415, - /* 1440 */ 343, 342, 418, 97, 101, 421, 422, 423, 424, 425, - /* 1450 */ 426, 333, 428, 188, 188, 20, 48, 433, 335, 435, - /* 1460 */ 335, 343, 20, 439, 440, 415, 339, 370, 418, 339, - /* 1470 */ 410, 421, 422, 423, 424, 425, 426, 20, 428, 382, - /* 1480 */ 20, 384, 350, 433, 384, 435, 344, 350, 370, 439, - /* 1490 */ 440, 20, 402, 53, 350, 344, 350, 350, 350, 342, - /* 1500 */ 382, 347, 384, 350, 335, 347, 342, 335, 370, 370, - /* 1510 */ 370, 370, 415, 382, 203, 418, 382, 382, 421, 422, - /* 1520 */ 423, 424, 425, 426, 414, 428, 370, 100, 412, 410, - /* 1530 */ 370, 370, 186, 415, 188, 348, 418, 370, 370, 421, - /* 1540 */ 422, 423, 424, 425, 426, 370, 428, 370, 192, 348, - /* 1550 */ 191, 433, 190, 435, 409, 333, 382, 439, 440, 261, - /* 1560 */ 384, 215, 216, 408, 456, 343, 342, 260, 397, 407, - /* 1570 */ 473, 392, 392, 382, 228, 229, 230, 231, 232, 233, - /* 1580 */ 234, 397, 456, 269, 177, 382, 382, 459, 271, 270, - /* 1590 */ 255, 458, 370, 278, 343, 417, 251, 475, 275, 20, - /* 1600 */ 273, 420, 344, 342, 382, 455, 384, 20, 395, 397, - /* 1610 */ 382, 348, 348, 397, 382, 170, 348, 382, 382, 382, - /* 1620 */ 394, 333, 456, 382, 454, 343, 453, 348, 100, 100, - /* 1630 */ 438, 343, 382, 342, 360, 366, 36, 415, 348, 336, - /* 1640 */ 418, 335, 398, 421, 422, 423, 424, 425, 426, 333, - /* 1650 */ 428, 469, 374, 468, 411, 433, 404, 435, 370, 343, - /* 1660 */ 364, 439, 440, 364, 364, 331, 0, 0, 398, 0, - /* 1670 */ 382, 349, 384, 42, 0, 35, 208, 35, 35, 35, - /* 1680 */ 208, 333, 0, 35, 35, 208, 370, 0, 208, 0, - /* 1690 */ 35, 343, 22, 0, 35, 195, 188, 186, 382, 0, - /* 1700 */ 384, 0, 0, 415, 0, 182, 418, 181, 0, 421, - /* 1710 */ 422, 423, 424, 425, 426, 0, 428, 0, 370, 47, - /* 1720 */ 0, 433, 0, 435, 0, 42, 0, 439, 440, 0, - /* 1730 */ 382, 415, 384, 0, 418, 0, 0, 421, 422, 423, - /* 1740 */ 424, 425, 426, 0, 428, 0, 155, 35, 0, 155, - /* 1750 */ 0, 435, 333, 0, 0, 439, 440, 139, 42, 0, - /* 1760 */ 0, 0, 343, 415, 0, 0, 418, 0, 0, 421, - /* 1770 */ 422, 423, 424, 425, 426, 0, 428, 0, 0, 0, - /* 1780 */ 333, 0, 0, 435, 0, 0, 0, 439, 440, 370, - /* 1790 */ 343, 0, 0, 0, 22, 0, 0, 0, 0, 333, - /* 1800 */ 0, 382, 22, 384, 48, 22, 48, 0, 59, 343, - /* 1810 */ 0, 35, 0, 0, 59, 333, 0, 370, 44, 14, - /* 1820 */ 47, 59, 0, 0, 0, 343, 39, 0, 39, 382, - /* 1830 */ 42, 384, 0, 39, 415, 177, 370, 418, 0, 0, - /* 1840 */ 421, 422, 423, 424, 425, 426, 47, 428, 382, 47, - /* 1850 */ 384, 40, 370, 0, 435, 0, 66, 375, 439, 440, - /* 1860 */ 35, 0, 415, 49, 382, 418, 384, 39, 421, 422, - /* 1870 */ 423, 424, 425, 426, 39, 428, 333, 35, 49, 0, - /* 1880 */ 35, 415, 49, 39, 418, 0, 343, 421, 422, 423, - /* 1890 */ 424, 425, 426, 35, 428, 49, 430, 415, 1, 333, - /* 1900 */ 418, 39, 0, 421, 422, 423, 424, 425, 426, 343, - /* 1910 */ 428, 464, 0, 370, 0, 0, 19, 109, 375, 35, - /* 1920 */ 22, 0, 107, 35, 35, 382, 35, 384, 35, 35, - /* 1930 */ 33, 35, 22, 0, 35, 22, 370, 51, 35, 44, - /* 1940 */ 35, 375, 35, 0, 44, 22, 49, 0, 382, 22, - /* 1950 */ 384, 35, 55, 56, 57, 58, 59, 35, 415, 0, - /* 1960 */ 0, 418, 0, 35, 421, 422, 423, 424, 425, 426, - /* 1970 */ 22, 428, 20, 35, 35, 35, 101, 100, 0, 35, - /* 1980 */ 100, 415, 0, 22, 418, 168, 22, 421, 422, 423, - /* 1990 */ 424, 425, 426, 333, 428, 189, 99, 0, 168, 102, - /* 2000 */ 0, 170, 193, 343, 168, 3, 100, 44, 256, 175, - /* 2010 */ 101, 100, 44, 48, 48, 98, 96, 44, 44, 101, - /* 2020 */ 101, 44, 47, 47, 100, 333, 100, 256, 3, 100, - /* 2030 */ 370, 134, 101, 101, 100, 343, 101, 44, 35, 47, - /* 2040 */ 35, 35, 382, 333, 384, 35, 35, 35, 47, 0, - /* 2050 */ 101, 101, 44, 343, 0, 0, 0, 256, 100, 39, - /* 2060 */ 47, 171, 370, 0, 110, 39, 169, 100, 250, 47, - /* 2070 */ 101, 174, 101, 169, 382, 415, 384, 44, 418, 100, - /* 2080 */ 370, 421, 422, 423, 424, 425, 426, 100, 428, 100, - /* 2090 */ 333, 194, 382, 237, 384, 100, 98, 98, 2, 22, - /* 2100 */ 343, 22, 215, 35, 47, 47, 35, 415, 101, 100, - /* 2110 */ 418, 35, 122, 421, 422, 423, 424, 425, 426, 100, - /* 2120 */ 428, 35, 101, 100, 100, 415, 111, 370, 418, 101, - /* 2130 */ 100, 421, 422, 423, 424, 425, 426, 333, 428, 382, - /* 2140 */ 101, 384, 217, 100, 35, 101, 100, 343, 101, 101, - /* 2150 */ 100, 100, 35, 101, 101, 44, 122, 122, 122, 333, - /* 2160 */ 100, 100, 100, 35, 100, 100, 22, 66, 65, 343, - /* 2170 */ 35, 35, 415, 44, 370, 418, 35, 35, 421, 422, - /* 2180 */ 423, 424, 425, 426, 333, 428, 382, 35, 384, 35, - /* 2190 */ 35, 35, 72, 94, 343, 35, 370, 35, 35, 22, - /* 2200 */ 35, 35, 35, 72, 35, 35, 333, 35, 382, 35, - /* 2210 */ 384, 35, 22, 35, 0, 35, 343, 49, 39, 415, - /* 2220 */ 0, 370, 418, 35, 49, 421, 422, 423, 424, 425, - /* 2230 */ 426, 39, 428, 382, 0, 384, 35, 39, 49, 0, - /* 2240 */ 35, 415, 49, 370, 418, 39, 0, 421, 422, 423, - /* 2250 */ 424, 425, 426, 35, 428, 382, 35, 384, 22, 21, - /* 2260 */ 476, 476, 22, 22, 21, 20, 415, 0, 476, 418, - /* 2270 */ 333, 476, 421, 422, 423, 424, 425, 426, 476, 428, - /* 2280 */ 343, 476, 476, 476, 476, 476, 476, 476, 415, 476, - /* 2290 */ 333, 418, 476, 476, 421, 422, 423, 424, 425, 426, - /* 2300 */ 343, 428, 476, 476, 476, 333, 476, 370, 476, 476, - /* 2310 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 382, - /* 2320 */ 476, 384, 476, 476, 476, 476, 476, 370, 476, 476, - /* 2330 */ 476, 476, 476, 476, 476, 476, 476, 333, 476, 382, - /* 2340 */ 476, 384, 370, 476, 476, 476, 476, 343, 476, 476, - /* 2350 */ 476, 476, 415, 476, 382, 418, 384, 476, 421, 422, - /* 2360 */ 423, 424, 425, 426, 476, 428, 476, 476, 476, 476, - /* 2370 */ 476, 476, 415, 476, 370, 418, 476, 476, 421, 422, - /* 2380 */ 423, 424, 425, 426, 476, 428, 382, 415, 384, 476, - /* 2390 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, - /* 2400 */ 428, 476, 476, 476, 476, 333, 476, 476, 476, 476, - /* 2410 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 415, - /* 2420 */ 476, 476, 418, 476, 333, 421, 422, 423, 424, 425, - /* 2430 */ 426, 476, 428, 476, 343, 476, 476, 476, 476, 476, - /* 2440 */ 476, 476, 370, 476, 476, 476, 476, 476, 476, 476, - /* 2450 */ 476, 476, 333, 476, 382, 476, 384, 476, 476, 476, - /* 2460 */ 476, 370, 343, 476, 476, 476, 476, 476, 476, 476, - /* 2470 */ 476, 333, 476, 382, 476, 384, 476, 476, 476, 476, - /* 2480 */ 476, 343, 476, 476, 476, 476, 476, 415, 476, 370, - /* 2490 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, - /* 2500 */ 428, 382, 476, 384, 476, 476, 415, 476, 370, 418, - /* 2510 */ 476, 476, 421, 422, 423, 424, 425, 426, 333, 428, - /* 2520 */ 382, 476, 384, 476, 476, 476, 476, 476, 343, 476, - /* 2530 */ 476, 476, 476, 476, 415, 476, 476, 418, 476, 476, - /* 2540 */ 421, 422, 423, 424, 425, 426, 476, 428, 476, 476, - /* 2550 */ 476, 476, 476, 415, 476, 370, 418, 476, 476, 421, - /* 2560 */ 422, 423, 424, 425, 426, 476, 428, 382, 476, 384, - /* 2570 */ 476, 476, 476, 476, 476, 333, 476, 476, 476, 476, - /* 2580 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 476, - /* 2590 */ 476, 333, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2600 */ 415, 343, 476, 418, 476, 476, 421, 422, 423, 424, - /* 2610 */ 425, 426, 370, 428, 476, 476, 476, 476, 476, 476, - /* 2620 */ 476, 476, 476, 476, 382, 476, 384, 476, 370, 476, - /* 2630 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2640 */ 382, 333, 384, 476, 476, 476, 476, 476, 476, 476, - /* 2650 */ 476, 343, 476, 476, 476, 476, 476, 415, 476, 476, - /* 2660 */ 418, 333, 476, 421, 422, 423, 424, 425, 426, 476, - /* 2670 */ 428, 343, 476, 415, 476, 476, 418, 476, 370, 421, - /* 2680 */ 422, 423, 424, 425, 426, 476, 428, 476, 476, 476, - /* 2690 */ 382, 476, 384, 476, 476, 476, 476, 476, 370, 476, - /* 2700 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, - /* 2710 */ 382, 476, 384, 476, 476, 476, 476, 476, 343, 476, - /* 2720 */ 476, 476, 476, 415, 476, 476, 418, 476, 476, 421, - /* 2730 */ 422, 423, 424, 425, 426, 476, 428, 476, 476, 476, - /* 2740 */ 476, 476, 33, 415, 476, 370, 418, 476, 476, 421, - /* 2750 */ 422, 423, 424, 425, 426, 476, 428, 382, 49, 384, - /* 2760 */ 476, 476, 476, 476, 55, 56, 57, 58, 59, 476, - /* 2770 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2780 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2790 */ 415, 476, 476, 418, 476, 476, 421, 422, 423, 424, - /* 2800 */ 425, 426, 476, 428, 476, 476, 476, 476, 99, 476, - /* 2810 */ 476, 102, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2820 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2830 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2840 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2850 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 333, 168, 44, + /* 1360 */ 101, 441, 101, 13, 44, 451, 13, 343, 254, 345, + /* 1370 */ 188, 402, 49, 417, 101, 404, 101, 101, 101, 101, + /* 1380 */ 42, 184, 389, 333, 20, 35, 392, 165, 35, 389, + /* 1390 */ 387, 20, 387, 343, 370, 345, 342, 342, 389, 387, + /* 1400 */ 98, 96, 188, 354, 353, 342, 382, 95, 384, 352, + /* 1410 */ 342, 342, 342, 20, 335, 48, 101, 335, 20, 339, + /* 1420 */ 370, 101, 339, 350, 411, 20, 384, 20, 344, 20, + /* 1430 */ 403, 350, 382, 344, 384, 350, 350, 350, 350, 342, + /* 1440 */ 416, 350, 418, 53, 335, 421, 422, 423, 424, 425, + /* 1450 */ 426, 347, 428, 347, 382, 333, 382, 433, 382, 435, + /* 1460 */ 370, 342, 370, 439, 440, 343, 416, 345, 418, 370, + /* 1470 */ 370, 421, 422, 423, 424, 425, 426, 370, 428, 335, + /* 1480 */ 370, 333, 370, 433, 203, 435, 370, 370, 370, 439, + /* 1490 */ 440, 343, 370, 370, 100, 348, 192, 191, 190, 384, + /* 1500 */ 342, 348, 261, 392, 382, 410, 384, 413, 415, 411, + /* 1510 */ 409, 382, 456, 408, 333, 260, 392, 382, 370, 382, + /* 1520 */ 456, 382, 402, 397, 343, 269, 459, 397, 177, 458, + /* 1530 */ 382, 456, 384, 271, 270, 255, 402, 475, 416, 273, + /* 1540 */ 418, 278, 275, 421, 422, 423, 424, 425, 426, 251, + /* 1550 */ 428, 370, 343, 20, 344, 433, 454, 435, 420, 342, + /* 1560 */ 20, 439, 440, 382, 416, 384, 418, 395, 402, 421, + /* 1570 */ 422, 423, 424, 425, 426, 348, 428, 469, 348, 366, + /* 1580 */ 455, 433, 397, 435, 382, 453, 382, 439, 440, 382, + /* 1590 */ 382, 333, 397, 382, 170, 468, 348, 416, 382, 418, + /* 1600 */ 394, 343, 421, 422, 423, 424, 425, 426, 100, 428, + /* 1610 */ 100, 343, 438, 348, 433, 382, 435, 374, 348, 36, + /* 1620 */ 439, 440, 333, 336, 342, 335, 405, 412, 370, 360, + /* 1630 */ 331, 0, 343, 398, 398, 364, 364, 349, 364, 0, + /* 1640 */ 382, 0, 384, 42, 0, 35, 208, 35, 35, 35, + /* 1650 */ 333, 208, 0, 35, 35, 208, 0, 208, 0, 370, + /* 1660 */ 343, 35, 0, 0, 22, 35, 195, 188, 186, 0, + /* 1670 */ 0, 382, 0, 384, 416, 182, 418, 181, 0, 421, + /* 1680 */ 422, 423, 424, 425, 426, 333, 428, 370, 0, 0, + /* 1690 */ 47, 433, 0, 435, 0, 343, 42, 439, 440, 382, + /* 1700 */ 0, 384, 0, 0, 0, 416, 0, 418, 0, 0, + /* 1710 */ 421, 422, 423, 424, 425, 426, 0, 428, 35, 155, + /* 1720 */ 0, 155, 370, 0, 435, 0, 0, 0, 439, 440, + /* 1730 */ 0, 0, 0, 416, 382, 418, 384, 0, 421, 422, + /* 1740 */ 423, 424, 425, 426, 0, 428, 0, 0, 0, 0, + /* 1750 */ 0, 0, 435, 0, 42, 0, 439, 440, 0, 0, + /* 1760 */ 0, 0, 0, 22, 0, 0, 0, 139, 416, 48, + /* 1770 */ 418, 0, 0, 421, 422, 423, 424, 425, 426, 333, + /* 1780 */ 428, 48, 0, 59, 0, 22, 0, 435, 59, 343, + /* 1790 */ 22, 439, 440, 0, 59, 0, 42, 14, 0, 333, + /* 1800 */ 44, 0, 35, 0, 47, 0, 177, 0, 0, 343, + /* 1810 */ 333, 0, 39, 0, 0, 40, 370, 47, 39, 39, + /* 1820 */ 343, 47, 35, 0, 39, 35, 49, 49, 382, 0, + /* 1830 */ 384, 39, 35, 39, 0, 35, 370, 49, 49, 0, + /* 1840 */ 0, 39, 66, 0, 0, 109, 107, 370, 382, 22, + /* 1850 */ 384, 35, 0, 35, 35, 35, 35, 35, 22, 382, + /* 1860 */ 35, 384, 416, 35, 418, 35, 0, 421, 422, 423, + /* 1870 */ 424, 425, 426, 35, 428, 22, 44, 44, 0, 22, + /* 1880 */ 35, 435, 416, 0, 418, 22, 440, 421, 422, 423, + /* 1890 */ 424, 425, 426, 416, 428, 418, 333, 51, 421, 422, + /* 1900 */ 423, 424, 425, 426, 35, 428, 343, 333, 0, 35, + /* 1910 */ 0, 35, 0, 22, 20, 35, 35, 343, 101, 35, + /* 1920 */ 0, 193, 333, 22, 100, 35, 100, 0, 462, 189, + /* 1930 */ 0, 22, 343, 370, 0, 3, 44, 256, 101, 98, + /* 1940 */ 44, 464, 100, 48, 370, 382, 48, 384, 100, 375, + /* 1950 */ 96, 170, 101, 47, 47, 44, 382, 101, 384, 370, + /* 1960 */ 100, 100, 100, 44, 168, 101, 175, 168, 100, 44, + /* 1970 */ 256, 382, 168, 384, 101, 3, 101, 44, 256, 416, + /* 1980 */ 35, 418, 35, 35, 421, 422, 423, 424, 425, 426, + /* 1990 */ 416, 428, 418, 333, 35, 421, 422, 423, 424, 425, + /* 2000 */ 426, 35, 428, 343, 35, 416, 101, 418, 333, 47, + /* 2010 */ 421, 422, 423, 424, 425, 426, 1, 428, 343, 430, + /* 2020 */ 44, 47, 101, 0, 0, 0, 0, 39, 100, 47, + /* 2030 */ 370, 100, 0, 39, 19, 375, 473, 101, 101, 100, + /* 2040 */ 100, 171, 382, 100, 384, 370, 169, 47, 33, 100, + /* 2050 */ 375, 110, 44, 98, 98, 2, 250, 382, 22, 384, + /* 2060 */ 100, 100, 215, 100, 49, 101, 101, 333, 101, 47, + /* 2070 */ 55, 56, 57, 58, 59, 100, 416, 343, 418, 100, + /* 2080 */ 47, 421, 422, 423, 424, 425, 426, 237, 428, 22, + /* 2090 */ 101, 416, 100, 418, 111, 35, 421, 422, 423, 424, + /* 2100 */ 425, 426, 333, 428, 370, 35, 217, 101, 101, 100, + /* 2110 */ 35, 100, 343, 101, 99, 35, 382, 102, 384, 100, + /* 2120 */ 35, 35, 101, 122, 122, 122, 44, 100, 35, 101, + /* 2130 */ 100, 100, 122, 22, 100, 66, 65, 100, 35, 370, + /* 2140 */ 35, 35, 35, 35, 35, 35, 72, 35, 94, 134, + /* 2150 */ 416, 382, 418, 384, 22, 421, 422, 423, 424, 425, + /* 2160 */ 426, 44, 428, 35, 333, 35, 35, 72, 35, 35, + /* 2170 */ 35, 35, 35, 35, 343, 35, 35, 22, 35, 0, + /* 2180 */ 35, 39, 0, 35, 169, 416, 0, 418, 49, 174, + /* 2190 */ 421, 422, 423, 424, 425, 426, 49, 428, 39, 35, + /* 2200 */ 49, 370, 39, 0, 35, 49, 39, 0, 35, 194, + /* 2210 */ 35, 0, 22, 382, 21, 384, 22, 22, 21, 20, + /* 2220 */ 476, 333, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2230 */ 476, 343, 333, 476, 476, 476, 476, 476, 476, 476, + /* 2240 */ 476, 476, 343, 476, 476, 476, 476, 416, 476, 418, + /* 2250 */ 476, 333, 421, 422, 423, 424, 425, 426, 370, 428, + /* 2260 */ 476, 343, 476, 476, 476, 476, 476, 476, 476, 370, + /* 2270 */ 382, 476, 384, 476, 476, 476, 476, 476, 476, 476, + /* 2280 */ 476, 382, 476, 384, 476, 476, 476, 476, 370, 476, + /* 2290 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2300 */ 382, 476, 384, 476, 416, 476, 418, 476, 476, 421, + /* 2310 */ 422, 423, 424, 425, 426, 416, 428, 418, 333, 476, + /* 2320 */ 421, 422, 423, 424, 425, 426, 476, 428, 343, 476, + /* 2330 */ 476, 476, 476, 476, 416, 476, 418, 333, 476, 421, + /* 2340 */ 422, 423, 424, 425, 426, 476, 428, 343, 476, 476, + /* 2350 */ 476, 476, 476, 333, 476, 370, 476, 476, 476, 476, + /* 2360 */ 476, 476, 476, 343, 476, 476, 476, 382, 333, 384, + /* 2370 */ 476, 476, 476, 476, 370, 476, 476, 476, 343, 476, + /* 2380 */ 476, 476, 476, 476, 333, 476, 382, 476, 384, 476, + /* 2390 */ 370, 476, 476, 476, 343, 476, 476, 476, 476, 476, + /* 2400 */ 476, 416, 382, 418, 384, 370, 421, 422, 423, 424, + /* 2410 */ 425, 426, 476, 428, 476, 476, 476, 382, 476, 384, + /* 2420 */ 416, 370, 418, 476, 476, 421, 422, 423, 424, 425, + /* 2430 */ 426, 476, 428, 382, 476, 384, 416, 476, 418, 476, + /* 2440 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2450 */ 476, 416, 333, 418, 476, 476, 421, 422, 423, 424, + /* 2460 */ 425, 426, 343, 428, 476, 476, 476, 416, 476, 418, + /* 2470 */ 476, 476, 421, 422, 423, 424, 425, 426, 476, 428, + /* 2480 */ 476, 476, 476, 333, 476, 476, 476, 476, 476, 370, + /* 2490 */ 476, 476, 476, 343, 476, 476, 476, 476, 476, 476, + /* 2500 */ 476, 382, 476, 384, 476, 476, 476, 476, 476, 476, + /* 2510 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2520 */ 370, 476, 476, 476, 476, 476, 476, 476, 343, 476, + /* 2530 */ 476, 476, 382, 476, 384, 416, 476, 418, 476, 476, + /* 2540 */ 421, 422, 423, 424, 425, 426, 476, 428, 476, 333, + /* 2550 */ 476, 476, 476, 476, 476, 370, 476, 476, 476, 343, + /* 2560 */ 476, 476, 476, 476, 476, 476, 416, 382, 418, 384, + /* 2570 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2580 */ 476, 476, 476, 476, 476, 476, 370, 476, 476, 476, + /* 2590 */ 476, 476, 476, 476, 476, 476, 476, 476, 382, 476, + /* 2600 */ 384, 416, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2610 */ 425, 426, 476, 428, 476, 333, 476, 476, 476, 476, + /* 2620 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 476, + /* 2630 */ 476, 476, 416, 476, 418, 476, 333, 421, 422, 423, + /* 2640 */ 424, 425, 426, 476, 428, 476, 343, 476, 476, 476, + /* 2650 */ 476, 476, 370, 476, 476, 476, 476, 476, 476, 476, + /* 2660 */ 476, 476, 333, 476, 382, 476, 384, 476, 476, 476, + /* 2670 */ 476, 476, 343, 370, 476, 476, 476, 476, 476, 476, + /* 2680 */ 476, 333, 476, 476, 476, 382, 476, 384, 476, 476, + /* 2690 */ 476, 343, 476, 476, 476, 476, 476, 476, 416, 370, + /* 2700 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2710 */ 428, 382, 333, 384, 476, 476, 476, 476, 370, 416, + /* 2720 */ 476, 418, 343, 476, 421, 422, 423, 424, 425, 426, + /* 2730 */ 382, 428, 384, 476, 476, 476, 476, 476, 333, 476, + /* 2740 */ 476, 476, 476, 476, 476, 416, 476, 418, 343, 370, + /* 2750 */ 421, 422, 423, 424, 425, 426, 476, 428, 476, 476, + /* 2760 */ 476, 382, 476, 384, 416, 476, 418, 476, 476, 421, + /* 2770 */ 422, 423, 424, 425, 426, 370, 428, 476, 476, 476, + /* 2780 */ 476, 476, 476, 476, 476, 476, 476, 382, 476, 384, + /* 2790 */ 476, 476, 476, 476, 476, 416, 476, 418, 476, 476, + /* 2800 */ 421, 422, 423, 424, 425, 426, 333, 428, 476, 476, + /* 2810 */ 476, 476, 476, 476, 476, 476, 343, 476, 476, 476, + /* 2820 */ 476, 416, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2830 */ 425, 426, 476, 428, 476, 476, 476, 476, 476, 476, + /* 2840 */ 476, 476, 476, 370, 476, 476, 476, 476, 476, 476, + /* 2850 */ 476, 476, 476, 476, 476, 382, 476, 384, 476, 476, /* 2860 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2870 */ 476, 476, 476, 476, 476, 476, 476, 476, 169, 170, - /* 2880 */ 476, 476, 476, 174, 476, 176, 476, 476, 476, 476, - /* 2890 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2900 */ 476, 476, 476, 194, 476, 476, 476, 476, 476, 476, - /* 2910 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2920 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2930 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2940 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2950 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2960 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2970 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2980 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2990 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3000 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3010 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3020 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3030 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3040 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3050 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3060 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3070 */ 476, 476, 476, 330, 330, 330, 330, 330, 330, 330, + /* 2870 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2880 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 416, + /* 2890 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2900 */ 476, 428, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2910 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2920 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2930 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2940 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2950 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2960 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2970 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2980 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2990 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3000 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3010 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3020 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3030 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3040 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3050 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3060 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3070 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3080 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3090 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3100 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, @@ -836,205 +836,205 @@ static const YYCODETYPE yy_lookahead[] = { /* 3200 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3210 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3220 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, - /* 3230 */ 330, 330, 330, 330, + /* 3230 */ 330, 330, }; -#define YY_SHIFT_COUNT (761) +#define YY_SHIFT_COUNT (763) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2709) +#define YY_SHIFT_MAX (2211) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 550, 0, 224, 0, 449, 449, 449, 449, 449, 449, /* 10 */ 449, 449, 449, 449, 449, 449, 673, 897, 897, 1121, /* 20 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, /* 30 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 148, - /* 50 */ 368, 93, 64, 140, 147, 322, 147, 64, 64, 1346, - /* 60 */ 1346, 1346, 147, 1346, 1346, 314, 147, 76, 369, 111, - /* 70 */ 111, 369, 656, 656, 18, 327, 7, 7, 111, 111, - /* 80 */ 111, 111, 111, 111, 111, 145, 111, 111, 248, 76, - /* 90 */ 111, 111, 363, 111, 76, 111, 145, 111, 145, 76, - /* 100 */ 111, 111, 76, 111, 76, 76, 76, 111, 343, 223, - /* 110 */ 189, 189, 405, 114, 443, 443, 443, 443, 443, 443, - /* 120 */ 443, 443, 443, 443, 443, 443, 443, 443, 443, 443, - /* 130 */ 443, 443, 443, 672, 483, 18, 327, 293, 253, 253, - /* 140 */ 253, 217, 622, 622, 293, 523, 523, 523, 248, 460, - /* 150 */ 354, 76, 585, 76, 585, 585, 604, 670, 35, 35, - /* 160 */ 35, 35, 35, 35, 35, 35, 1897, 636, 15, 167, - /* 170 */ 16, 297, 515, 252, 454, 454, 373, 842, 389, 484, - /* 180 */ 1058, 1163, 862, 892, 922, 983, 45, 922, 878, 727, - /* 190 */ 948, 1136, 1347, 1214, 1358, 1382, 1358, 1239, 1390, 1390, - /* 200 */ 1358, 1239, 1239, 1313, 1329, 1390, 1338, 1390, 1390, 1390, - /* 210 */ 1435, 1408, 1435, 1408, 1442, 248, 1457, 248, 1460, 1471, - /* 220 */ 248, 1460, 248, 248, 248, 1390, 248, 1440, 1440, 1435, - /* 230 */ 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, - /* 240 */ 76, 1390, 1435, 585, 585, 585, 1311, 1427, 1442, 343, - /* 250 */ 1356, 1359, 1457, 343, 1362, 1390, 1382, 1382, 585, 1298, - /* 260 */ 1307, 585, 1298, 1307, 585, 585, 76, 1314, 1407, 1298, - /* 270 */ 1317, 1319, 1335, 1136, 1315, 1323, 1327, 1345, 523, 1579, - /* 280 */ 1390, 1460, 343, 343, 1587, 1307, 585, 585, 585, 585, - /* 290 */ 585, 1307, 585, 1445, 343, 604, 343, 523, 1528, 1529, - /* 300 */ 585, 670, 1390, 343, 1600, 1435, 2904, 2904, 2904, 2904, - /* 310 */ 2904, 2904, 2904, 2904, 2904, 34, 2709, 238, 590, 198, - /* 320 */ 539, 973, 794, 486, 1064, 925, 1030, 48, 48, 48, - /* 330 */ 48, 48, 48, 48, 48, 48, 831, 612, 65, 65, - /* 340 */ 549, 737, 700, 891, 262, 448, 1025, 717, 531, 531, - /* 350 */ 962, 751, 906, 962, 962, 962, 1202, 1015, 513, 1219, - /* 360 */ 1008, 1101, 1194, 1104, 1113, 1149, 1159, 381, 1229, 1248, - /* 370 */ 1259, 1267, 1069, 1196, 1221, 1217, 1231, 1233, 1234, 1143, - /* 380 */ 836, 1152, 1165, 1260, 1262, 1273, 1276, 1277, 1318, 1281, - /* 390 */ 1283, 1208, 1284, 1238, 1320, 1321, 1326, 1328, 1336, 1343, - /* 400 */ 671, 1265, 1266, 1255, 1289, 1325, 1366, 1666, 1667, 1669, - /* 410 */ 1631, 1674, 1640, 1468, 1642, 1643, 1644, 1472, 1682, 1648, - /* 420 */ 1649, 1477, 1687, 1480, 1689, 1655, 1699, 1670, 1693, 1659, - /* 430 */ 1500, 1508, 1511, 1701, 1702, 1704, 1523, 1526, 1708, 1715, - /* 440 */ 1672, 1717, 1720, 1722, 1683, 1724, 1726, 1729, 1733, 1735, - /* 450 */ 1736, 1743, 1745, 1591, 1712, 1748, 1594, 1750, 1753, 1754, - /* 460 */ 1764, 1765, 1767, 1768, 1775, 1777, 1778, 1779, 1781, 1782, - /* 470 */ 1784, 1785, 1786, 1716, 1759, 1760, 1761, 1791, 1792, 1793, - /* 480 */ 1772, 1795, 1796, 1797, 1618, 1798, 1800, 1780, 1756, 1783, - /* 490 */ 1758, 1807, 1749, 1776, 1810, 1755, 1812, 1762, 1813, 1816, - /* 500 */ 1788, 1787, 1774, 1773, 1799, 1805, 1802, 1822, 1811, 1789, - /* 510 */ 1823, 1824, 1827, 1794, 1658, 1832, 1838, 1839, 1790, 1853, - /* 520 */ 1855, 1825, 1814, 1828, 1861, 1842, 1829, 1835, 1879, 1845, - /* 530 */ 1833, 1844, 1885, 1858, 1846, 1862, 1902, 1912, 1914, 1915, - /* 540 */ 1808, 1815, 1884, 1898, 1921, 1888, 1889, 1891, 1893, 1894, - /* 550 */ 1896, 1899, 1895, 1900, 1903, 1905, 1910, 1907, 1933, 1913, - /* 560 */ 1943, 1923, 1886, 1947, 1927, 1916, 1959, 1922, 1960, 1928, - /* 570 */ 1962, 1948, 1952, 1938, 1939, 1940, 1875, 1877, 1978, 1817, - /* 580 */ 1880, 1809, 1944, 1961, 1982, 1806, 1964, 1830, 1831, 1997, - /* 590 */ 2000, 1836, 1834, 2002, 1963, 1752, 1906, 1909, 1911, 1965, - /* 600 */ 1917, 1966, 1920, 1918, 1968, 1973, 1919, 1924, 1926, 1929, - /* 610 */ 1931, 1974, 1975, 1976, 1934, 1977, 1771, 1932, 1935, 2025, - /* 620 */ 1993, 1801, 2003, 2005, 2006, 2010, 2011, 2012, 1949, 1950, - /* 630 */ 1992, 1818, 2008, 2001, 2049, 2054, 2055, 2056, 1958, 2020, - /* 640 */ 1773, 2013, 1967, 1969, 1971, 1979, 1987, 1890, 1989, 2063, - /* 650 */ 2026, 1904, 1995, 1954, 1773, 2022, 2033, 1998, 1856, 1999, - /* 660 */ 2096, 2077, 1887, 2009, 2007, 2019, 2021, 2023, 2028, 2057, - /* 670 */ 2024, 2030, 2058, 2039, 2079, 1925, 2043, 2015, 2044, 2068, - /* 680 */ 2071, 2046, 2047, 2076, 2050, 2048, 2086, 2051, 2052, 2109, - /* 690 */ 2060, 2053, 2117, 2061, 1990, 2034, 2035, 2036, 2062, 2111, - /* 700 */ 2064, 2128, 2065, 2111, 2111, 2144, 2101, 2103, 2135, 2136, - /* 710 */ 2141, 2142, 2152, 2154, 2155, 2156, 2120, 2099, 2129, 2160, - /* 720 */ 2162, 2163, 2177, 2165, 2166, 2167, 2131, 1895, 2169, 1900, - /* 730 */ 2170, 2172, 2174, 2176, 2190, 2178, 2214, 2180, 2168, 2179, - /* 740 */ 2220, 2188, 2175, 2192, 2234, 2201, 2189, 2198, 2239, 2205, - /* 750 */ 2193, 2206, 2246, 2218, 2221, 2267, 2236, 2238, 2240, 2241, - /* 760 */ 2243, 2245, + /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 113, + /* 50 */ 165, 815, 547, 666, 112, 215, 112, 547, 547, 117, + /* 60 */ 117, 117, 112, 117, 117, 6, 112, 128, 222, 162, + /* 70 */ 162, 222, 236, 236, 190, 271, 247, 247, 162, 162, + /* 80 */ 162, 162, 162, 162, 162, 296, 162, 162, 4, 128, + /* 90 */ 162, 162, 303, 162, 128, 162, 296, 162, 296, 128, + /* 100 */ 162, 162, 128, 162, 128, 128, 128, 162, 70, 223, + /* 110 */ 189, 189, 226, 130, 446, 446, 446, 446, 446, 446, + /* 120 */ 446, 446, 446, 446, 446, 446, 446, 446, 446, 446, + /* 130 */ 446, 446, 446, 433, 315, 190, 271, 594, 522, 522, + /* 140 */ 522, 520, 13, 13, 594, 583, 583, 583, 4, 98, + /* 150 */ 397, 128, 659, 128, 659, 659, 545, 848, 755, 755, + /* 160 */ 755, 755, 755, 755, 755, 755, 2015, 424, 501, 611, + /* 170 */ 15, 212, 717, 230, 307, 307, 452, 483, 277, 946, + /* 180 */ 868, 929, 697, 998, 612, 773, 586, 612, 1128, 949, + /* 190 */ 1026, 1114, 1323, 1197, 1338, 1364, 1338, 1222, 1371, 1371, + /* 200 */ 1338, 1222, 1222, 1302, 1305, 1371, 1312, 1371, 1371, 1371, + /* 210 */ 1393, 1367, 1393, 1367, 1398, 4, 1405, 4, 1407, 1409, + /* 220 */ 4, 1407, 4, 4, 4, 1371, 4, 1390, 1390, 1393, + /* 230 */ 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + /* 240 */ 128, 1371, 1393, 659, 659, 659, 1281, 1394, 1398, 70, + /* 250 */ 1304, 1306, 1405, 70, 1308, 1114, 1371, 1364, 1364, 659, + /* 260 */ 1241, 1255, 659, 1241, 1255, 659, 659, 128, 1256, 1351, + /* 270 */ 1241, 1262, 1264, 1280, 1114, 1263, 1267, 1266, 1298, 583, + /* 280 */ 1533, 1114, 1371, 1407, 70, 70, 1540, 1255, 659, 659, + /* 290 */ 659, 659, 659, 1255, 659, 1424, 70, 545, 70, 583, + /* 300 */ 1508, 1510, 659, 848, 1371, 70, 1583, 1393, 2902, 2902, + /* 310 */ 2902, 2902, 2902, 2902, 2902, 2902, 2902, 34, 229, 51, + /* 320 */ 1157, 31, 584, 406, 425, 805, 820, 835, 793, 48, + /* 330 */ 48, 48, 48, 48, 48, 48, 48, 48, 580, 910, + /* 340 */ 3, 3, 166, 359, 32, 29, 367, 670, 37, 781, + /* 350 */ 646, 646, 592, 308, 712, 592, 592, 592, 1146, 885, + /* 360 */ 1113, 238, 1131, 1037, 1165, 1077, 1078, 1096, 1103, 1047, + /* 370 */ 1170, 1191, 1212, 1219, 977, 1145, 1150, 1177, 1152, 1180, + /* 380 */ 1196, 950, 1030, 939, 1190, 1198, 1223, 1231, 1232, 1234, + /* 390 */ 1259, 1260, 1261, 287, 1273, 416, 1275, 1276, 1277, 1278, + /* 400 */ 1315, 1320, 1173, 1182, 1214, 1350, 1353, 759, 1252, 1631, + /* 410 */ 1639, 1641, 1601, 1644, 1610, 1438, 1612, 1613, 1614, 1443, + /* 420 */ 1652, 1618, 1619, 1447, 1656, 1449, 1658, 1626, 1662, 1642, + /* 430 */ 1663, 1630, 1471, 1479, 1482, 1669, 1670, 1672, 1493, 1496, + /* 440 */ 1678, 1688, 1643, 1689, 1692, 1694, 1654, 1700, 1702, 1703, + /* 450 */ 1704, 1706, 1708, 1709, 1716, 1564, 1683, 1720, 1566, 1723, + /* 460 */ 1725, 1726, 1727, 1730, 1731, 1732, 1737, 1744, 1746, 1747, + /* 470 */ 1748, 1749, 1750, 1751, 1753, 1712, 1755, 1758, 1759, 1760, + /* 480 */ 1761, 1762, 1741, 1764, 1765, 1766, 1628, 1771, 1772, 1763, + /* 490 */ 1721, 1768, 1733, 1782, 1724, 1767, 1784, 1729, 1786, 1735, + /* 500 */ 1793, 1795, 1754, 1773, 1756, 1757, 1770, 1783, 1774, 1798, + /* 510 */ 1775, 1779, 1801, 1803, 1805, 1780, 1629, 1807, 1808, 1811, + /* 520 */ 1776, 1813, 1814, 1787, 1777, 1785, 1823, 1790, 1778, 1792, + /* 530 */ 1829, 1797, 1788, 1794, 1834, 1800, 1789, 1802, 1839, 1840, + /* 540 */ 1843, 1844, 1736, 1739, 1816, 1827, 1852, 1818, 1819, 1820, + /* 550 */ 1821, 1822, 1825, 1828, 1832, 1833, 1830, 1838, 1836, 1845, + /* 560 */ 1866, 1853, 1878, 1857, 1846, 1883, 1863, 1869, 1908, 1874, + /* 570 */ 1910, 1876, 1912, 1891, 1894, 1880, 1881, 1884, 1817, 1824, + /* 580 */ 1920, 1796, 1826, 1728, 1890, 1901, 1927, 1740, 1909, 1799, + /* 590 */ 1781, 1930, 1934, 1804, 1791, 1932, 1892, 1681, 1842, 1837, + /* 600 */ 1848, 1895, 1841, 1898, 1854, 1851, 1896, 1911, 1856, 1860, + /* 610 */ 1861, 1862, 1864, 1919, 1906, 1907, 1868, 1925, 1714, 1873, + /* 620 */ 1875, 1972, 1933, 1722, 1945, 1947, 1948, 1959, 1966, 1969, + /* 630 */ 1905, 1921, 1962, 1806, 1976, 1974, 2023, 2024, 2025, 2026, + /* 640 */ 1928, 1988, 1757, 1982, 1931, 1936, 1937, 1939, 1940, 1870, + /* 650 */ 1943, 2032, 1994, 1877, 1949, 1941, 1757, 2000, 2008, 1955, + /* 660 */ 1850, 1956, 2053, 2036, 1847, 1960, 1964, 1961, 1965, 1963, + /* 670 */ 1967, 2022, 1975, 1979, 2033, 1989, 2067, 1889, 1992, 1983, + /* 680 */ 2006, 2060, 2070, 2009, 2007, 2075, 2011, 2012, 2080, 2019, + /* 690 */ 2021, 2085, 2027, 2028, 2086, 2030, 2001, 2002, 2003, 2010, + /* 700 */ 2031, 2082, 2034, 2093, 2037, 2082, 2082, 2111, 2069, 2071, + /* 710 */ 2103, 2105, 2106, 2107, 2108, 2109, 2110, 2112, 2074, 2054, + /* 720 */ 2117, 2128, 2130, 2131, 2132, 2133, 2134, 2135, 2095, 1832, + /* 730 */ 2136, 1833, 2137, 2138, 2140, 2141, 2155, 2143, 2179, 2145, + /* 740 */ 2139, 2142, 2182, 2148, 2147, 2159, 2186, 2164, 2151, 2163, + /* 750 */ 2203, 2169, 2156, 2167, 2207, 2173, 2175, 2211, 2190, 2193, + /* 760 */ 2194, 2195, 2197, 2199, }; -#define YY_REDUCE_COUNT (314) -#define YY_REDUCE_MIN (-397) -#define YY_REDUCE_MAX (2375) +#define YY_REDUCE_COUNT (316) +#define YY_REDUCE_MIN (-445) +#define YY_REDUCE_MAX (2473) static const short yy_reduce_ofst[] = { - /* 0 */ -156, 364, -82, 588, -20, 614, 804, 890, 1024, 1050, - /* 10 */ 1118, 1222, 1288, 1316, 1348, 1419, 174, -333, 154, 450, - /* 20 */ 674, 1447, 1097, 1482, 1466, 1543, 1566, 1660, 1692, 1710, - /* 30 */ 1757, 1804, 1826, 1851, 1873, 1937, 1957, 1972, 2004, 2072, - /* 40 */ 2091, 2119, 2138, 2185, 2242, 2258, 2308, 2328, 2375, 383, - /* 50 */ 270, -397, 83, -395, 371, 416, 554, 75, 501, 235, - /* 60 */ 786, 827, -139, 683, 834, -195, -165, -359, -80, -172, - /* 70 */ -13, -383, -254, -207, -213, 149, -52, 290, -152, -5, - /* 80 */ 263, 376, 407, 413, 453, 82, 508, 593, 462, 271, - /* 90 */ 610, 631, 275, 657, 374, 725, 301, 745, 372, -170, - /* 100 */ 748, 796, 399, 808, -83, 421, 605, 810, 260, -319, - /* 110 */ -338, -338, -293, -327, 94, 237, 433, 502, 534, 669, - /* 120 */ 693, 712, 802, 807, 809, 813, 815, 833, 849, 854, - /* 130 */ 865, 867, 868, -275, -251, -131, 355, 467, -251, 437, - /* 140 */ 559, 452, 445, 479, 726, -334, -85, 719, 681, 168, - /* 150 */ 736, 268, 134, 805, 797, 799, 742, 850, -369, 387, - /* 160 */ 499, 580, 713, 832, 913, 713, 637, 927, 485, 899, - /* 170 */ 818, 835, 949, 839, 941, 945, 940, 940, 980, 939, - /* 180 */ 1032, 1028, 994, 984, 933, 933, 919, 933, 947, 938, - /* 190 */ 940, 974, 976, 996, 1012, 1011, 1018, 1022, 1070, 1071, - /* 200 */ 1026, 1027, 1037, 1063, 1073, 1089, 1083, 1094, 1096, 1099, - /* 210 */ 1123, 1127, 1125, 1130, 1060, 1132, 1100, 1137, 1142, 1090, - /* 220 */ 1144, 1151, 1146, 1147, 1148, 1157, 1153, 1154, 1158, 1169, - /* 230 */ 1138, 1139, 1140, 1141, 1156, 1160, 1161, 1167, 1168, 1175, - /* 240 */ 1177, 1164, 1172, 1131, 1134, 1135, 1110, 1116, 1119, 1187, - /* 250 */ 1145, 1155, 1176, 1201, 1162, 1224, 1179, 1180, 1174, 1108, - /* 260 */ 1171, 1191, 1126, 1184, 1203, 1204, 940, 1128, 1133, 1166, - /* 270 */ 1150, 1170, 1173, 1178, 1122, 1182, 1185, 933, 1251, 1181, - /* 280 */ 1261, 1258, 1263, 1264, 1213, 1212, 1228, 1232, 1235, 1236, - /* 290 */ 1237, 1216, 1241, 1226, 1268, 1269, 1279, 1282, 1192, 1278, - /* 300 */ 1250, 1274, 1291, 1290, 1303, 1306, 1252, 1243, 1244, 1270, - /* 310 */ 1296, 1299, 1300, 1322, 1334, + /* 0 */ -195, 364, 220, 588, 614, 804, 890, 1024, 1050, 1122, + /* 10 */ 1148, 1181, 1258, 1289, 1317, 1352, 153, 674, -332, 1446, + /* 20 */ 1466, 1477, 1563, 1574, 1589, 1660, 1675, 1734, 1769, 1831, + /* 30 */ 1888, 1899, 1918, 1985, 2004, 2020, 2035, 2051, 2119, 2150, + /* 40 */ 2185, 2216, 2282, 2303, 2329, 2348, 2379, 2405, 2473, -301, + /* 50 */ -202, 251, -104, 515, 798, 800, 830, 544, 632, -92, + /* 60 */ 377, 643, 253, 383, 780, -445, -276, -178, -330, -294, + /* 70 */ 45, -221, -337, -242, 115, -203, 175, 206, -306, 50, + /* 80 */ 297, 499, 523, 591, 593, 152, 495, 599, -159, -232, + /* 90 */ 209, 603, 191, 723, 319, 730, 349, 746, 430, 239, + /* 100 */ 797, 802, 366, 808, 413, 578, 681, 817, 564, -333, + /* 110 */ -430, -430, 114, -247, -29, 22, 108, 445, 473, 475, + /* 120 */ 482, 557, 617, 638, 640, 644, 649, 675, 686, 719, + /* 130 */ 722, 760, 822, -44, -20, -8, 211, 371, -20, 338, + /* 140 */ 354, 257, 306, 506, 498, 410, 662, 765, 344, 158, + /* 150 */ 268, 598, 368, 386, 699, 785, 627, 811, -363, 292, + /* 160 */ 573, 596, 624, 732, 739, 624, 847, 828, 921, 892, + /* 170 */ 812, 819, 943, 833, 927, 933, 922, 922, 952, 918, + /* 180 */ 985, 984, 947, 942, 882, 882, 871, 882, 920, 914, + /* 190 */ 922, 969, 956, 971, 993, 994, 1000, 1003, 1054, 1055, + /* 200 */ 1009, 1005, 1012, 1049, 1051, 1063, 1057, 1068, 1069, 1070, + /* 210 */ 1079, 1080, 1082, 1083, 1013, 1073, 1042, 1081, 1084, 1027, + /* 220 */ 1085, 1089, 1086, 1087, 1088, 1097, 1091, 1104, 1106, 1109, + /* 230 */ 1090, 1092, 1099, 1100, 1107, 1110, 1112, 1116, 1117, 1118, + /* 240 */ 1123, 1119, 1144, 1072, 1074, 1076, 1093, 1094, 1098, 1147, + /* 250 */ 1095, 1101, 1115, 1153, 1105, 1120, 1158, 1111, 1124, 1129, + /* 260 */ 1056, 1126, 1135, 1064, 1130, 1137, 1139, 922, 1067, 1071, + /* 270 */ 1075, 1125, 1102, 1132, 1134, 1062, 1108, 1127, 882, 1209, + /* 280 */ 1138, 1166, 1217, 1210, 1227, 1230, 1172, 1185, 1202, 1204, + /* 290 */ 1207, 1208, 1211, 1195, 1216, 1206, 1248, 1213, 1265, 1268, + /* 300 */ 1174, 1243, 1233, 1269, 1282, 1270, 1287, 1290, 1221, 1215, + /* 310 */ 1235, 1236, 1271, 1272, 1274, 1288, 1299, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 10 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 20 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 30 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 40 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 50 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 60 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 70 */ 1719, 1719, 1719, 1719, 1992, 1719, 1719, 1719, 1719, 1719, - /* 80 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1801, 1719, - /* 90 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 100 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1799, 1985, - /* 110 */ 2206, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 120 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 130 */ 1719, 1719, 1719, 1719, 2218, 1719, 1719, 1719, 2218, 2218, - /* 140 */ 2218, 1799, 2178, 2178, 1719, 1719, 1719, 1719, 1801, 2050, - /* 150 */ 1719, 1719, 1719, 1719, 1719, 1719, 1920, 1719, 1719, 1719, - /* 160 */ 1719, 1719, 1944, 1719, 1719, 1719, 2044, 1719, 1719, 2243, - /* 170 */ 2299, 1719, 1719, 2246, 1719, 1719, 1719, 1719, 1719, 1997, - /* 180 */ 1719, 1719, 1874, 2233, 2210, 2224, 2283, 2211, 2208, 2227, - /* 190 */ 1719, 2237, 1719, 2031, 1990, 1719, 1990, 1987, 1719, 1719, - /* 200 */ 1990, 1987, 1987, 1863, 1859, 1719, 1857, 1719, 1719, 1719, - /* 210 */ 1719, 1766, 1719, 1766, 1719, 1801, 1719, 1801, 1719, 1719, - /* 220 */ 1801, 1719, 1801, 1801, 1801, 1719, 1801, 1779, 1779, 1719, - /* 230 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 240 */ 1719, 1719, 1719, 1719, 1719, 1719, 2062, 2048, 1719, 1799, - /* 250 */ 2042, 2040, 1719, 1799, 2038, 1719, 1719, 1719, 1719, 2254, - /* 260 */ 2252, 1719, 2254, 2252, 1719, 1719, 1719, 2268, 2264, 2254, - /* 270 */ 2272, 2270, 2239, 2237, 2302, 2289, 2285, 2224, 1719, 1719, - /* 280 */ 1719, 1719, 1799, 1799, 1719, 2252, 1719, 1719, 1719, 1719, - /* 290 */ 1719, 2252, 1719, 1719, 1799, 1719, 1799, 1719, 1719, 1890, - /* 300 */ 1719, 1719, 1719, 1799, 1751, 1719, 2033, 2053, 2015, 2015, - /* 310 */ 1923, 1923, 1923, 1802, 1724, 1719, 1719, 1719, 1719, 1719, - /* 320 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2267, 2266, 2133, - /* 330 */ 1719, 2182, 2181, 2180, 2171, 2132, 1886, 1719, 2131, 2130, - /* 340 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2006, 2005, - /* 350 */ 2124, 1719, 1719, 2125, 2123, 2122, 1719, 1719, 1719, 1719, - /* 360 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 370 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 380 */ 2286, 2290, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2207, - /* 390 */ 1719, 1719, 1719, 2106, 1719, 1719, 1719, 1719, 1719, 1719, - /* 400 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 410 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 420 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 430 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 440 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 450 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 460 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 470 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 480 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 490 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 500 */ 1719, 1719, 1756, 2111, 1719, 1719, 1719, 1719, 1719, 1719, - /* 510 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 520 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 530 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 540 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 550 */ 1719, 1719, 1840, 1839, 1719, 1719, 1719, 1719, 1719, 1719, - /* 560 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 570 */ 1719, 1719, 1719, 1719, 1719, 1719, 2115, 1719, 1719, 1719, - /* 580 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 590 */ 1719, 1719, 1719, 2282, 2240, 1719, 1719, 1719, 1719, 1719, - /* 600 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 610 */ 1719, 1719, 1719, 2106, 1719, 2265, 1719, 1719, 2280, 1719, - /* 620 */ 2284, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2217, 2213, - /* 630 */ 1719, 1719, 2209, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 640 */ 2114, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 650 */ 1719, 1719, 1719, 1719, 2105, 1719, 2168, 1719, 1719, 1719, - /* 660 */ 2202, 1719, 1719, 2153, 1719, 1719, 1719, 1719, 1719, 1719, - /* 670 */ 1719, 1719, 1719, 2115, 1719, 2118, 1719, 1719, 1719, 1719, - /* 680 */ 1719, 1917, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 690 */ 1719, 1719, 1719, 1719, 1902, 1900, 1899, 1898, 1719, 1930, - /* 700 */ 1719, 1719, 1719, 1926, 1925, 1719, 1719, 1719, 1719, 1719, - /* 710 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1820, 1719, - /* 720 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1812, 1719, 1811, - /* 730 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 740 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 750 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 760 */ 1719, 1719, + /* 0 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 10 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 20 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 30 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 40 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 50 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 60 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 70 */ 1721, 1721, 1721, 1721, 1994, 1721, 1721, 1721, 1721, 1721, + /* 80 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1803, 1721, + /* 90 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 100 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1801, 1987, + /* 110 */ 2208, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 120 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 130 */ 1721, 1721, 1721, 1721, 2220, 1721, 1721, 1721, 2220, 2220, + /* 140 */ 2220, 1801, 2180, 2180, 1721, 1721, 1721, 1721, 1803, 2052, + /* 150 */ 1721, 1721, 1721, 1721, 1721, 1721, 1922, 1721, 1721, 1721, + /* 160 */ 1721, 1721, 1946, 1721, 1721, 1721, 2046, 1721, 1721, 2245, + /* 170 */ 2301, 1721, 1721, 2248, 1721, 1721, 1721, 1721, 1721, 1999, + /* 180 */ 1721, 1721, 1876, 2235, 2212, 2226, 2285, 2213, 2210, 2229, + /* 190 */ 1721, 2239, 1721, 2033, 1992, 1721, 1992, 1989, 1721, 1721, + /* 200 */ 1992, 1989, 1989, 1865, 1861, 1721, 1859, 1721, 1721, 1721, + /* 210 */ 1721, 1768, 1721, 1768, 1721, 1803, 1721, 1803, 1721, 1721, + /* 220 */ 1803, 1721, 1803, 1803, 1803, 1721, 1803, 1781, 1781, 1721, + /* 230 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 240 */ 1721, 1721, 1721, 1721, 1721, 1721, 2064, 2050, 1721, 1801, + /* 250 */ 2044, 2042, 1721, 1801, 2040, 2239, 1721, 1721, 1721, 1721, + /* 260 */ 2256, 2254, 1721, 2256, 2254, 1721, 1721, 1721, 2270, 2266, + /* 270 */ 2256, 2274, 2272, 2241, 2239, 2304, 2291, 2287, 2226, 1721, + /* 280 */ 1721, 2239, 1721, 1721, 1801, 1801, 1721, 2254, 1721, 1721, + /* 290 */ 1721, 1721, 1721, 2254, 1721, 1721, 1801, 1721, 1801, 1721, + /* 300 */ 1721, 1892, 1721, 1721, 1721, 1801, 1753, 1721, 2035, 2055, + /* 310 */ 2017, 2017, 1925, 1925, 1925, 1804, 1726, 1721, 1721, 1721, + /* 320 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2269, + /* 330 */ 2268, 2135, 1721, 2184, 2183, 2182, 2173, 2134, 1888, 1721, + /* 340 */ 2133, 2132, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 350 */ 2008, 2007, 2126, 1721, 1721, 2127, 2125, 2124, 1721, 1721, + /* 360 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 370 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 380 */ 1721, 1721, 2288, 2292, 1721, 1721, 1721, 1721, 1721, 1721, + /* 390 */ 1721, 2209, 1721, 1721, 1721, 2108, 1721, 1721, 1721, 1721, + /* 400 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 410 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 420 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 430 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 440 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 450 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 460 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 470 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 480 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 490 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 500 */ 1721, 1721, 1721, 1721, 1758, 2113, 1721, 1721, 1721, 1721, + /* 510 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 520 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 530 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 540 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 550 */ 1721, 1721, 1721, 1721, 1842, 1841, 1721, 1721, 1721, 1721, + /* 560 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 570 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2117, 1721, + /* 580 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 590 */ 1721, 1721, 1721, 1721, 1721, 2284, 2242, 1721, 1721, 1721, + /* 600 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 610 */ 1721, 1721, 1721, 1721, 1721, 2108, 1721, 2267, 1721, 1721, + /* 620 */ 2282, 1721, 2286, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 630 */ 2219, 2215, 1721, 1721, 2211, 1721, 1721, 1721, 1721, 1721, + /* 640 */ 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 650 */ 1721, 1721, 1721, 1721, 1721, 1721, 2107, 1721, 2170, 1721, + /* 660 */ 1721, 1721, 2204, 1721, 1721, 2155, 1721, 1721, 1721, 1721, + /* 670 */ 1721, 1721, 1721, 1721, 1721, 2117, 1721, 2120, 1721, 1721, + /* 680 */ 1721, 1721, 1721, 1919, 1721, 1721, 1721, 1721, 1721, 1721, + /* 690 */ 1721, 1721, 1721, 1721, 1721, 1721, 1904, 1902, 1901, 1900, + /* 700 */ 1721, 1932, 1721, 1721, 1721, 1928, 1927, 1721, 1721, 1721, + /* 710 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 720 */ 1822, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1814, + /* 730 */ 1721, 1813, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 740 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 750 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 760 */ 1721, 1721, 1721, 1721, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1873,22 +1873,22 @@ static const char *const yyTokenName[] = { /* 399 */ "func", /* 400 */ "sma_func_name", /* 401 */ "query_or_subquery", - /* 402 */ "cgroup_name", - /* 403 */ "analyze_opt", - /* 404 */ "explain_options", - /* 405 */ "insert_query", - /* 406 */ "or_replace_opt", - /* 407 */ "agg_func_opt", - /* 408 */ "bufsize_opt", - /* 409 */ "language_opt", - /* 410 */ "stream_name", - /* 411 */ "stream_options", - /* 412 */ "col_list_opt", - /* 413 */ "tag_def_or_ref_opt", - /* 414 */ "subtable_opt", - /* 415 */ "expression", - /* 416 */ "dnode_list", - /* 417 */ "where_clause_opt", + /* 402 */ "where_clause_opt", + /* 403 */ "cgroup_name", + /* 404 */ "analyze_opt", + /* 405 */ "explain_options", + /* 406 */ "insert_query", + /* 407 */ "or_replace_opt", + /* 408 */ "agg_func_opt", + /* 409 */ "bufsize_opt", + /* 410 */ "language_opt", + /* 411 */ "stream_name", + /* 412 */ "stream_options", + /* 413 */ "col_list_opt", + /* 414 */ "tag_def_or_ref_opt", + /* 415 */ "subtable_opt", + /* 416 */ "expression", + /* 417 */ "dnode_list", /* 418 */ "literal_func", /* 419 */ "literal_list", /* 420 */ "table_alias", @@ -2254,8 +2254,8 @@ static const char *const yyRuleName[] = { /* 297 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", /* 298 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", /* 299 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 300 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 301 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 300 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt", + /* 301 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt", /* 302 */ "cmd ::= DROP TOPIC exists_opt topic_name", /* 303 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", /* 304 */ "cmd ::= DESC full_table_name", @@ -2695,12 +2695,12 @@ static void yy_destructor( case 398: /* sma_stream_opt */ case 399: /* func */ case 401: /* query_or_subquery */ - case 404: /* explain_options */ - case 405: /* insert_query */ - case 411: /* stream_options */ - case 414: /* subtable_opt */ - case 415: /* expression */ - case 417: /* where_clause_opt */ + case 402: /* where_clause_opt */ + case 405: /* explain_options */ + case 406: /* insert_query */ + case 412: /* stream_options */ + case 415: /* subtable_opt */ + case 416: /* expression */ case 418: /* literal_func */ case 421: /* expr_or_subquery */ case 422: /* pseudo_column */ @@ -2745,7 +2745,7 @@ static void yy_destructor( case 332: /* alter_account_options */ case 334: /* alter_account_option */ case 352: /* speed_opt */ - case 408: /* bufsize_opt */ + case 409: /* bufsize_opt */ { } @@ -2760,9 +2760,9 @@ static void yy_destructor( case 392: /* column_alias */ case 395: /* index_name */ case 400: /* sma_func_name */ - case 402: /* cgroup_name */ - case 409: /* language_opt */ - case 410: /* stream_name */ + case 403: /* cgroup_name */ + case 410: /* language_opt */ + case 411: /* stream_name */ case 420: /* table_alias */ case 426: /* star_func */ case 428: /* noarg_func */ @@ -2791,9 +2791,9 @@ static void yy_destructor( case 347: /* force_opt */ case 348: /* not_exists_opt */ case 350: /* exists_opt */ - case 403: /* analyze_opt */ - case 406: /* or_replace_opt */ - case 407: /* agg_func_opt */ + case 404: /* analyze_opt */ + case 407: /* or_replace_opt */ + case 408: /* agg_func_opt */ case 451: /* set_quantifier_opt */ { @@ -2814,9 +2814,9 @@ static void yy_destructor( case 380: /* rollup_func_list */ case 390: /* tag_list_opt */ case 396: /* func_list */ - case 412: /* col_list_opt */ - case 413: /* tag_def_or_ref_opt */ - case 416: /* dnode_list */ + case 413: /* col_list_opt */ + case 414: /* tag_def_or_ref_opt */ + case 417: /* dnode_list */ case 419: /* literal_list */ case 427: /* star_func_para_list */ case 429: /* other_para_list */ @@ -3455,8 +3455,8 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 330, /* (297) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ 330, /* (298) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ 330, /* (299) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - 330, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - 330, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + 330, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt */ + 330, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt */ 330, /* (302) cmd ::= DROP TOPIC exists_opt topic_name */ 330, /* (303) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ 330, /* (304) cmd ::= DESC full_table_name */ @@ -3464,39 +3464,39 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 330, /* (306) cmd ::= RESET QUERY CACHE */ 330, /* (307) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ 330, /* (308) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ - 403, /* (309) analyze_opt ::= */ - 403, /* (310) analyze_opt ::= ANALYZE */ - 404, /* (311) explain_options ::= */ - 404, /* (312) explain_options ::= explain_options VERBOSE NK_BOOL */ - 404, /* (313) explain_options ::= explain_options RATIO NK_FLOAT */ + 404, /* (309) analyze_opt ::= */ + 404, /* (310) analyze_opt ::= ANALYZE */ + 405, /* (311) explain_options ::= */ + 405, /* (312) explain_options ::= explain_options VERBOSE NK_BOOL */ + 405, /* (313) explain_options ::= explain_options RATIO NK_FLOAT */ 330, /* (314) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ 330, /* (315) cmd ::= DROP FUNCTION exists_opt function_name */ - 407, /* (316) agg_func_opt ::= */ - 407, /* (317) agg_func_opt ::= AGGREGATE */ - 408, /* (318) bufsize_opt ::= */ - 408, /* (319) bufsize_opt ::= BUFSIZE NK_INTEGER */ - 409, /* (320) language_opt ::= */ - 409, /* (321) language_opt ::= LANGUAGE NK_STRING */ - 406, /* (322) or_replace_opt ::= */ - 406, /* (323) or_replace_opt ::= OR REPLACE */ + 408, /* (316) agg_func_opt ::= */ + 408, /* (317) agg_func_opt ::= AGGREGATE */ + 409, /* (318) bufsize_opt ::= */ + 409, /* (319) bufsize_opt ::= BUFSIZE NK_INTEGER */ + 410, /* (320) language_opt ::= */ + 410, /* (321) language_opt ::= LANGUAGE NK_STRING */ + 407, /* (322) or_replace_opt ::= */ + 407, /* (323) or_replace_opt ::= OR REPLACE */ 330, /* (324) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ 330, /* (325) cmd ::= DROP STREAM exists_opt stream_name */ - 412, /* (326) col_list_opt ::= */ - 412, /* (327) col_list_opt ::= NK_LP col_name_list NK_RP */ - 413, /* (328) tag_def_or_ref_opt ::= */ - 413, /* (329) tag_def_or_ref_opt ::= tags_def */ - 413, /* (330) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ - 411, /* (331) stream_options ::= */ - 411, /* (332) stream_options ::= stream_options TRIGGER AT_ONCE */ - 411, /* (333) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - 411, /* (334) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - 411, /* (335) stream_options ::= stream_options WATERMARK duration_literal */ - 411, /* (336) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - 411, /* (337) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - 411, /* (338) stream_options ::= stream_options DELETE_MARK duration_literal */ - 411, /* (339) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - 414, /* (340) subtable_opt ::= */ - 414, /* (341) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + 413, /* (326) col_list_opt ::= */ + 413, /* (327) col_list_opt ::= NK_LP col_name_list NK_RP */ + 414, /* (328) tag_def_or_ref_opt ::= */ + 414, /* (329) tag_def_or_ref_opt ::= tags_def */ + 414, /* (330) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + 412, /* (331) stream_options ::= */ + 412, /* (332) stream_options ::= stream_options TRIGGER AT_ONCE */ + 412, /* (333) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + 412, /* (334) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + 412, /* (335) stream_options ::= stream_options WATERMARK duration_literal */ + 412, /* (336) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + 412, /* (337) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + 412, /* (338) stream_options ::= stream_options DELETE_MARK duration_literal */ + 412, /* (339) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + 415, /* (340) subtable_opt ::= */ + 415, /* (341) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ 330, /* (342) cmd ::= KILL CONNECTION NK_INTEGER */ 330, /* (343) cmd ::= KILL QUERY NK_STRING */ 330, /* (344) cmd ::= KILL TRANSACTION NK_INTEGER */ @@ -3505,13 +3505,13 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 330, /* (347) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ 330, /* (348) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ 330, /* (349) cmd ::= SPLIT VGROUP NK_INTEGER */ - 416, /* (350) dnode_list ::= DNODE NK_INTEGER */ - 416, /* (351) dnode_list ::= dnode_list DNODE NK_INTEGER */ + 417, /* (350) dnode_list ::= DNODE NK_INTEGER */ + 417, /* (351) dnode_list ::= dnode_list DNODE NK_INTEGER */ 330, /* (352) cmd ::= DELETE FROM full_table_name where_clause_opt */ 330, /* (353) cmd ::= query_or_subquery */ 330, /* (354) cmd ::= insert_query */ - 405, /* (355) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - 405, /* (356) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + 406, /* (355) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + 406, /* (356) insert_query ::= INSERT INTO full_table_name query_or_subquery */ 333, /* (357) literal ::= NK_INTEGER */ 333, /* (358) literal ::= NK_FLOAT */ 333, /* (359) literal ::= NK_STRING */ @@ -3545,26 +3545,26 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 392, /* (387) column_alias ::= NK_ID */ 335, /* (388) user_name ::= NK_ID */ 344, /* (389) topic_name ::= NK_ID */ - 410, /* (390) stream_name ::= NK_ID */ - 402, /* (391) cgroup_name ::= NK_ID */ + 411, /* (390) stream_name ::= NK_ID */ + 403, /* (391) cgroup_name ::= NK_ID */ 395, /* (392) index_name ::= NK_ID */ 421, /* (393) expr_or_subquery ::= expression */ - 415, /* (394) expression ::= literal */ - 415, /* (395) expression ::= pseudo_column */ - 415, /* (396) expression ::= column_reference */ - 415, /* (397) expression ::= function_expression */ - 415, /* (398) expression ::= case_when_expression */ - 415, /* (399) expression ::= NK_LP expression NK_RP */ - 415, /* (400) expression ::= NK_PLUS expr_or_subquery */ - 415, /* (401) expression ::= NK_MINUS expr_or_subquery */ - 415, /* (402) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - 415, /* (403) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - 415, /* (404) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - 415, /* (405) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - 415, /* (406) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - 415, /* (407) expression ::= column_reference NK_ARROW NK_STRING */ - 415, /* (408) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - 415, /* (409) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + 416, /* (394) expression ::= literal */ + 416, /* (395) expression ::= pseudo_column */ + 416, /* (396) expression ::= column_reference */ + 416, /* (397) expression ::= function_expression */ + 416, /* (398) expression ::= case_when_expression */ + 416, /* (399) expression ::= NK_LP expression NK_RP */ + 416, /* (400) expression ::= NK_PLUS expr_or_subquery */ + 416, /* (401) expression ::= NK_MINUS expr_or_subquery */ + 416, /* (402) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + 416, /* (403) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + 416, /* (404) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + 416, /* (405) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + 416, /* (406) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + 416, /* (407) expression ::= column_reference NK_ARROW NK_STRING */ + 416, /* (408) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + 416, /* (409) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ 375, /* (410) expression_list ::= expr_or_subquery */ 375, /* (411) expression_list ::= expression_list NK_COMMA expr_or_subquery */ 423, /* (412) column_reference ::= column_name */ @@ -3670,8 +3670,8 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 460, /* (512) select_item ::= common_expression column_alias */ 460, /* (513) select_item ::= common_expression AS column_alias */ 460, /* (514) select_item ::= table_name NK_DOT NK_STAR */ - 417, /* (515) where_clause_opt ::= */ - 417, /* (516) where_clause_opt ::= WHERE search_condition */ + 402, /* (515) where_clause_opt ::= */ + 402, /* (516) where_clause_opt ::= WHERE search_condition */ 453, /* (517) partition_by_clause_opt ::= */ 453, /* (518) partition_by_clause_opt ::= PARTITION BY partition_list */ 461, /* (519) partition_list ::= partition_item */ @@ -4043,8 +4043,8 @@ static const signed char yyRuleInfoNRhs[] = { -6, /* (297) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -7, /* (298) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -9, /* (299) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - -7, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - -9, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + -8, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt */ + -10, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt */ -4, /* (302) cmd ::= DROP TOPIC exists_opt topic_name */ -7, /* (303) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -2, /* (304) cmd ::= DESC full_table_name */ @@ -5390,11 +5390,11 @@ static YYACTIONTYPE yy_reduce( case 299: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy369, &yymsp[-5].minor.yy113, &yymsp[0].minor.yy113, true); } break; - case 300: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy369, &yymsp[-3].minor.yy113, yymsp[0].minor.yy448, false); } + case 300: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-5].minor.yy369, &yymsp[-4].minor.yy113, yymsp[-1].minor.yy448, false, yymsp[0].minor.yy448); } break; - case 301: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy369, &yymsp[-5].minor.yy113, yymsp[0].minor.yy448, true); } + case 301: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-7].minor.yy369, &yymsp[-6].minor.yy113, yymsp[-1].minor.yy448, true, yymsp[0].minor.yy448); } break; case 302: /* cmd ::= DROP TOPIC exists_opt topic_name */ { pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 6a08193a39..a4e8bdd87a 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -1145,6 +1145,15 @@ TEST_F(ParserInitialCTest, createTopic) { setCreateTopicReq("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1); run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1"); clearCreateTopicReq(); + + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 as stable st1 where tag1 > 0", nullptr, "test", "st1"); + run("CREATE TOPIC IF NOT EXISTS tp1 AS STABLE st1 WHERE tag1 > 0"); + clearCreateTopicReq(); + + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 with meta as stable st1 where tag1 > 0", nullptr, "test", "st1", 1); + run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1 WHERE tag1 > 0"); + clearCreateTopicReq(); + } /* diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 9d8c170003..1260ea336c 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -449,6 +449,18 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } +void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType) { + int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; + for (int32_t i = 0; i < nums; ++i) { + if (0 == strcmp(pName, pMeta->schema[i].name)) { + *pType = (i < pMeta->tableInfo.numOfColumns) ? TCOL_TYPE_COLUMN : TCOL_TYPE_TAG; + return; + } + } + + *pType = TCOL_TYPE_NONE; +} + void freeVgInfo(SDBVgInfo* vgInfo) { if (NULL == vgInfo) { return; diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index 0bf0873e9f..85a23055a3 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -108,4 +108,14 @@ if $rows != 6 then return -1 endi +return -1 +sql create topic topic_stable_1 as stable stb where t1 > 0 +sql create topic topic_stable_1 as stable stb where t1 > 0 and t1 < 0 +sql create topic topic_stable_1 as stable stb where 1 > 0 +sql create topic topic_stable_1 as stable stb where last(t1) > 0 +sql create topic topic_stable_1 as stable stb where tbname is not null +sql create topic topic_stable_1 as stable stb where tbname > 'a' +sql create topic topic_stable_1 as stable stb where tbname > 0 and xx < 0 +sql create topic topic_stable_1 as stable stb where tbname > 0 and c1 < 0 + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 409656f7b6c81aa61aee935ba403825135565c9d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 10:05:54 +0800 Subject: [PATCH 089/187] fix(query): fix bug in iterating the table list. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5d970ce6c3..2eff4ff8da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3433,6 +3433,8 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { if (!hasNexTable) { return TSDB_CODE_SUCCESS; } + + pBlockScanInfo = pStatus->pTableIter; } initMemDataIterator(*pBlockScanInfo, pReader); From 74f753606b69019d5cd9d415551c3294bb51dfde Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 10:37:36 +0800 Subject: [PATCH 090/187] fix(query): add regression test case. --- tests/script/tsim/compute/last_row.sim | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/script/tsim/compute/last_row.sim b/tests/script/tsim/compute/last_row.sim index 2e060dc285..8e62fbffb5 100644 --- a/tests/script/tsim/compute/last_row.sim +++ b/tests/script/tsim/compute/last_row.sim @@ -213,4 +213,54 @@ if $rows != 2 then return -1 endi +print =======================> regresss bug in last_row query +sql drop database if exists db; +sql create database if not exists db vgroups 1 cachemodel 'both'; +sql create table db.stb (ts timestamp, c0 bigint) tags(t1 int); +sql insert into db.stb_0 using db.stb tags(1) values ('2023-11-23 19:06:40.000', 491173569); +sql insert into db.stb_2 using db.stb tags(3) values ('2023-11-25 19:30:00.000', 2080726142); +sql insert into db.stb_3 using db.stb tags(4) values ('2023-11-26 06:48:20.000', 1907405128); +sql insert into db.stb_4 using db.stb tags(5) values ('2023-11-24 22:56:40.000', 220783803); + +sql create table db.stb_1 using db.stb tags(2); +sql insert into db.stb_1 (ts) values('2023-11-26 13:11:40.000'); +sql insert into db.stb_1 (ts, c0) values('2023-11-26 13:11:39.000', 11); + +sql select tbname,ts,last_row(c0) from db.stb; +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data01 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +sql alter database db cachemodel 'none'; +sql reset query cache; +sql select tbname,last_row(c0, ts) from db.stb; + +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data02 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data01 != NULL then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From faaa9da88748d7b13d645bc2ca3ec42e92bd4359 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 26 May 2023 18:17:05 +0800 Subject: [PATCH 091/187] fix: sliding can't be 1a issue --- source/common/src/tglobal.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bef63d8a49..2a9c54c189 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -139,8 +139,8 @@ int32_t tsCompressColData = -1; // count/hyperloglog function always return values in case of all NULL data or Empty data set. int32_t tsCountAlwaysReturnValue = 1; -// 10 ms for sliding time, the value will changed in case of time precision changed -int32_t tsMinSlidingTime = 10; +// 1 ms for sliding time, the value will changed in case of time precision changed +int32_t tsMinSlidingTime = 1; // the maxinum number of distict query result int32_t tsMaxNumOfDistinctResults = 1000 * 10000; @@ -412,7 +412,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; From 50487255b33a79f2d63f6009db00426d8e799c88 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 26 May 2023 19:02:40 +0800 Subject: [PATCH 092/187] fix:put poll to push manager if wal not exist when offset is latest --- source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/tq/tqUtil.c | 61 ++++++----------------------- source/libs/executor/src/executor.c | 3 ++ 3 files changed, 16 insertions(+), 49 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ef36b8429a..e6ba5d8636 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -106,7 +106,6 @@ typedef struct { // STqPushHandle pushHandle; // push STqExecHandle execHandle; // exec SRpcMsg* msg; - int32_t noDataPollCnt; tq_handle_status status; } STqHandle; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 622cc5b049..33d2a238c5 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -16,7 +16,6 @@ #include "tq.h" #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) -#define NO_POLL_CNT 5 static int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp); @@ -88,15 +87,12 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { return 0; } -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, bool* pBlockReturned) { +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest) { uint64_t consumerId = pRequest->consumerId; STqOffsetVal reqOffset = pRequest->reqOffset; STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); int32_t vgId = TD_VID(pTq->pVnode); - *pBlockReturned = false; - // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. if (pOffset != NULL) { *pOffsetVal = pOffset->val; @@ -129,28 +125,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); - - tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.rspOffset.version); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); - tDeleteSMqDataRsp(&dataRsp); - - *pBlockReturned = true; - return code; - } else { - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); - tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - - *pBlockReturned = true; - return code; - } + // offset set to previous version when init + tqOffsetResetToLog(pOffsetVal, walGetLastVer(pTq->pVnode->pWal)); } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); @@ -173,25 +149,19 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); code = tqScanData(pTq, pHandle, &dataRsp, pOffset); - if(code != 0) { + if(code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { - if(pHandle->noDataPollCnt >= NO_POLL_CNT){ // send poll result to client if no data 5 times to avoid lost data - pHandle->noDataPollCnt = 0; - // lock - taosWLockLatch(&pTq->lock); - code = tqRegisterPushHandle(pTq, pHandle, pMsg); - taosWUnLockLatch(&pTq->lock); - tDeleteSMqDataRsp(&dataRsp); - return code; - } - else{ - pHandle->noDataPollCnt++; - } + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST || (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && + dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId)) { + // lock + taosWLockLatch(&pTq->lock); + code = tqRegisterPushHandle(pTq, pHandle, pMsg); + taosWUnLockLatch(&pTq->lock); + tDeleteSMqDataRsp(&dataRsp); + return code; } // NOTE: this pHandle->consumerId may have been changed already. @@ -326,16 +296,11 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ // 1. reset the offset if needed if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { // handle the reset offset cases, according to the consumer's choice. - bool blockReturned = false; - code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest); if (code != 0) { return code; } - // empty block returned, quit - if (blockReturned) { - return 0; - } } else { // use the consumer specified offset // the offset value can not be monotonious increase?? offset = reqOffset; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a73deffa52..fb0b9957f8 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1080,6 +1080,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume + char buf[80] = {0}; + tFormatOffset(buf, 80, &pTaskInfo->streamInfo.currentOffset); + qDebug("currentOffset:%s", buf); if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } From a3e214b9e8eb24b842ad22b96f09e83bcf9e2632 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 26 May 2023 19:17:04 +0800 Subject: [PATCH 093/187] fix:put poll to push manager if wal not exist when offset is latest --- include/libs/wal/wal.h | 3 ++- source/dnode/vnode/src/tq/tqUtil.c | 12 +++--------- source/libs/executor/src/executor.c | 3 --- source/libs/wal/src/walRef.c | 17 +++++++++-------- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 7e106eefde..7af218b78e 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -205,7 +205,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead); int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead); int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead); -SWalRef *walRefFirstVer(SWal *, SWalRef *); +void walRefFirstVer(SWal *, SWalRef *); +void walRefLastVer(SWal *, SWalRef *); SWalRef *walRefCommittedVer(SWal *); SWalRef *walOpenRef(SWal *); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 33d2a238c5..5eaf7b240b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -115,18 +115,12 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand tqOffsetResetToData(pOffsetVal, 0, 0); } } else { - pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); - if (pHandle->pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // offset set to previous version when init + walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - // offset set to previous version when init - tqOffsetResetToLog(pOffsetVal, walGetLastVer(pTq->pVnode->pWal)); + walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); + tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fb0b9957f8..a73deffa52 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1080,9 +1080,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume - char buf[80] = {0}; - tFormatOffset(buf, 80, &pTaskInfo->streamInfo.currentOffset); - qDebug("currentOffset:%s", buf); if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index 6aba661926..eb36389f1d 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -63,21 +63,22 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { return 0; } -SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) { - if (pRef == NULL) { - pRef = walOpenRef(pWal); - if (pRef == NULL) { - return NULL; - } - } +void walRefFirstVer(SWal *pWal, SWalRef *pRef) { taosThreadMutexLock(&pWal->mutex); int64_t ver = walGetFirstVer(pWal); pRef->refVer = ver; taosThreadMutexUnlock(&pWal->mutex); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver); +} - return pRef; +void walRefLastVer(SWal *pWal, SWalRef *pRef) { + taosThreadMutexLock(&pWal->mutex); + int64_t ver = walGetLastVer(pWal); + pRef->refVer = ver; + + taosThreadMutexUnlock(&pWal->mutex); + wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, ver); } SWalRef *walRefCommittedVer(SWal *pWal) { From 6368b27499bebda87ef4059d1b542ee9bb67c2c2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 29 May 2023 08:50:57 +0800 Subject: [PATCH 094/187] fix: sliding case issue --- tests/script/tsim/parser/sliding.sim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/script/tsim/parser/sliding.sim b/tests/script/tsim/parser/sliding.sim index 1cb4cb5340..7aa69ce9a9 100644 --- a/tests/script/tsim/parser/sliding.sim +++ b/tests/script/tsim/parser/sliding.sim @@ -450,10 +450,11 @@ endi print ====================>check boundary check crash at client side sql select count(*) from sliding_mt0 where ts>now and ts < now-1h; +sql select sum(c1) from sliding_tb0 interval(1a) sliding(1a); + print ========================query on super table print ========================error case -sql_error select sum(c1) from sliding_tb0 interval(1a) sliding(1a); sql_error select sum(c1) from sliding_tb0 interval(10a) sliding(12a); sql_error select sum(c1) from sliding_tb0 sliding(1n) interval(1y); sql_error select sum(c1) from sliding_tb0 interval(-1y) sliding(1n); From 43c0656df2fa0ff5f864185eed902cb8a2fa1d52 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 29 May 2023 10:45:26 +0800 Subject: [PATCH 095/187] fix: if block row num is less than definit min rows, set the bucket index to 0 --- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5d970ce6c3..371ea77b4e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -5242,6 +5242,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { } static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) { + if (numOfRows < startRow) { + return 0; + } int32_t bucketIndex = ((numOfRows - startRow) / bucketRange); if (bucketIndex == numOfBucket) { bucketIndex -= 1; From a71878d07fb94e184991259a3fd18a5745e4afd9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 11:46:59 +0800 Subject: [PATCH 096/187] feat: support create topic as stable with conditions --- include/common/tmsg.h | 2 +- include/libs/executor/executor.h | 2 +- source/dnode/mnode/impl/src/mndScheduler.c | 20 +++++++++++++++++--- source/dnode/mnode/impl/src/mndStb.c | 4 ++-- source/dnode/mnode/impl/src/mndTopic.c | 10 ++++++++-- source/dnode/vnode/src/tq/tq.c | 15 +++++++++------ source/dnode/vnode/src/tq/tqMeta.c | 10 ++++++---- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/executor/src/executil.c | 13 +++++++------ 9 files changed, 52 insertions(+), 26 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a7de03fbce..db46858cec 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2772,8 +2772,8 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { buf = taosDecodeString(buf, &pReq->qmsg); } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { - buf = taosDecodeString(buf, &pReq->qmsg); buf = taosDecodeFixedI64(buf, &pReq->suid); + buf = taosDecodeString(buf, &pReq->qmsg); } return (void*)buf; } diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index e3a75ecabc..d64b3dd193 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -82,7 +82,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); -int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* pTagCond, void* pTagIndexCond, SArray **tableList); +int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* node, SArray **tableList); /** * set the task Id, usually used by message queue process diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 0248a195db..39c771d111 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -538,7 +538,23 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL){ + SNode *pAst = NULL; + if (nodesStringToNode(pTopic->ast, &pAst) != 0) { + mError("topic:%s, failed to create since %s", pTopic->name, terrstr()); + return -1; + } + SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; + if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { + mError("failed to create topic:%s since %s", pTopic->name, terrstr()); + nodesDestroyNode(pAst); + return -1; + } + nodesDestroyNode(pAst); + } + + if(pPlan){ int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { qDestroyQueryPlan(pPlan); @@ -579,7 +595,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib mDebug("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId); - if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) { + if (pSubplan) { int32_t msgLen; pSubplan->execNode.epSet = pVgEp->epSet; @@ -591,8 +607,6 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - } else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ - pVgEp->qmsg = taosStrdup(pTopic->ast); } else { pVgEp->qmsg = taosStrdup(""); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8b708c3e0f..2b54eb6389 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1205,7 +1205,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, mInfo("topic:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, subType:%d sql:%s", pTopic->name, stbFullName, suid, colId, pTopic->subType, pTopic->sql); - if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { + if (pTopic->ast == NULL) { sdbRelease(pSdb, pTopic); continue; } @@ -2247,7 +2247,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, } } - if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { + if (pTopic->ast == NULL) { sdbRelease(pSdb, pTopic); continue; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 7d71aae3f4..863055922a 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -420,6 +420,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * mError("failed to create topic:%s since %s", pCreate->name, terrstr()); taosMemoryFree(topicObj.ast); taosMemoryFree(topicObj.sql); + nodesDestroyNode(pAst); return -1; } @@ -427,6 +428,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (topicObj.ntbColIds == NULL) { taosMemoryFree(topicObj.ast); taosMemoryFree(topicObj.sql); + nodesDestroyNode(pAst); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -442,6 +444,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); taosMemoryFree(topicObj.ast); taosMemoryFree(topicObj.sql); + nodesDestroyNode(pAst); return -1; } @@ -462,8 +465,11 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.stbUid = pStb->uid; mndReleaseStb(pMnode, pStb); - topicObj.ast = taosStrdup(pCreate->ast); - topicObj.astLen = strlen(pCreate->ast) + 1; + if(pCreate->ast != NULL){ + qDebugL("topic:%s ast %s", topicObj.name, pCreate->ast); + topicObj.ast = taosStrdup(pCreate->ast); + topicObj.astLen = strlen(pCreate->ast) + 1; + } } /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/ /*topicObj.ast = NULL;*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index be3d6bc614..30c52d9fc2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -518,19 +518,22 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->execHandle.execTb.qmsg = req.qmsg; req.qmsg = NULL; - if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { - tqError("nodesStringToNode error in sub stable, since %s", terrstr()); - return -1; + if(strcmp(pHandle->execHandle.execTb.qmsg, "") != 0) { + if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s, vgId:%d, subkey:%s consumer:0x%" PRIx64, terrstr(), + pVnode->config.vgId, req.subKey, pHandle->consumerId); + return -1; + } } SArray* tbUidList = NULL; - ret = qGetTableList(req.suid, pVnode->pMeta, pVnode, pHandle->execHandle.execTb.node, NULL, &tbUidList); + ret = qGetTableList(req.suid, pVnode->pMeta, pVnode, pHandle->execHandle.execTb.node, &tbUidList); if(ret != TDB_CODE_SUCCESS) { - tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, req.subKey, pHandle->consumerId); + tqError("qGetTableList error:%d vgId:%d, subkey:%s consumer:0x%" PRIx64, ret, pVnode->config.vgId, req.subKey, pHandle->consumerId); taosArrayDestroy(tbUidList); goto end; } - tqDebug("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, req.suid); + tqDebug("tq try to get ctb for stb subscribe, vgId:%d, subkey:%s consumer:0x%" PRIx64 " suid:%" PRId64, pVnode->config.vgId, req.subKey, pHandle->consumerId, req.suid); pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index ee7af5b2bf..e93efcd3cc 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -338,13 +338,15 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - if (nodesStringToNode(handle.execHandle.execTb.qmsg, &handle.execHandle.execTb.node) != 0) { - tqError("nodesStringToNode error in sub stable, since %s", terrstr()); - return -1; + if(strcmp(handle.execHandle.execTb.qmsg, "") != 0) { + if (nodesStringToNode(handle.execHandle.execTb.qmsg, &handle.execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s", terrstr()); + return -1; + } } SArray* tbUidList = NULL; - int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, handle.execHandle.execTb.node, NULL, &tbUidList); + int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle.subKey, handle.consumerId); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 38f5307384..820a8b0eba 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1041,7 +1041,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* list = NULL; - int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, pTqHandle->execHandle.execTb.node, NULL, &list); + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); taosArrayDestroy(list); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1fb35ae271..fb889692ef 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1122,13 +1122,14 @@ _end: return code; } -int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* pTagCond, void* pTagIndexCond, SArray **tableList){ - SScanPhysiNode node = {0}; - node.suid = suid; - node.uid = suid; - node.tableType = TSDB_SUPER_TABLE; +int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* node, SArray **tableList){ + SSubplan *pSubplan = (SSubplan *)node; + SScanPhysiNode pNode = {0}; + pNode.suid = suid; + pNode.uid = suid; + pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); - int code = getTableList(metaHandle, pVnode, &node, pTagCond, pTagIndexCond, pTableListInfo, "qGetTableList"); + int code = getTableList(metaHandle, pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList"); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From 60588af69476567940b6668e3c526c650a772104 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 29 May 2023 13:33:51 +0800 Subject: [PATCH 097/187] fix: add topic query --- source/libs/parser/src/parTranslater.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6de1bd4b85..b858658709 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5882,6 +5882,7 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt strcpy(realTable->table.tableAlias, pStmt->subSTbName); *pSelect = createSelectStmtImpl(true, pProjection, (SNode*)realTable); ((SSelectStmt*)*pSelect)->pWhere = nodesCloneNode(pStmt->pWhere); + pCxt->pParseCxt->topicQuery = true; code = translateQuery(pCxt, *pSelect); } From 97e0e0ea0da6e709d36b98e92017c7aef5b61ca6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 14:35:59 +0800 Subject: [PATCH 098/187] fix:add test case --- tests/system-test/7-tmq/stbFilterWhere.py | 159 ++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 tests/system-test/7-tmq/stbFilterWhere.py diff --git a/tests/system-test/7-tmq/stbFilterWhere.py b/tests/system-test/7-tmq/stbFilterWhere.py new file mode 100644 index 0000000000..baf042d71c --- /dev/null +++ b/tests/system-test/7-tmq/stbFilterWhere.py @@ -0,0 +1,159 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + tmqCom.initConsumerTable() + tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica']) + tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict["dbName"])) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + return + + def tmqCaseError(self, topicName, condition): + tdLog.printNoPrefix("======== test case error: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + tdLog.info("create topics from stb with column filter") + topicString = "create topic %s as stable %s.%s where %s" %(topicName, paraDict['dbName'], paraDict['stbName'], condition) + tdLog.info("create topic sql: %s"%topicString) + tdSql.error(topicString) + + def tmqCase(self, topicName, condition): + tdLog.printNoPrefix("======== test case: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with tag filter") + topicString = "create topic %s as stable %s.%s where %s" %(topicName, paraDict['dbName'], paraDict['stbName'], condition) + tdLog.info("create topic sql: %s"%topicString) + tdSql.execute(topicString) + + queryString = "select * from %s.%s where %s" %(paraDict['dbName'], paraDict['stbName'], condition) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicName + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + tdLog.printNoPrefix("======== test case end ...... ") + + def run(self): + tdSql.prepare() + self.prepareTestEnv() + self.tmqCaseError("t1", "c1 = 4 and t1 = 3") + self.tmqCase("t2", "2 > 1") + self.tmqCase("t3", "t4 = 'beijing'") + self.tmqCase("t4", "t4 > t3") + self.tmqCase("t5", "t3 = t4") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 3bbf5405c6f9d80521c76b065166be765d0ed435 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 29 May 2023 14:36:29 +0800 Subject: [PATCH 099/187] fix(query): invalid buf page --- source/libs/executor/inc/executil.h | 3 +++ source/util/src/tpagedbuf.c | 1 + 2 files changed, 4 insertions(+) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 5b248f9a90..0f169b0c2e 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -135,6 +135,9 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); + if (!bufPage) { + return NULL; + } if (forUpdate) { setBufPageDirty(bufPage, true); } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index fa8b5d33b7..218a47ea62 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -482,6 +482,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); if (*pInfo != *pi) { + terrno = TSDB_CODE_QRY_SYS_ERROR; uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id); return NULL; } From e36acd0a63b2396da2a277d944a441085bf92cec Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 14:45:06 +0800 Subject: [PATCH 100/187] fix:add test case --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4065ac5bee..44a08da90e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -75,6 +75,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/db.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqError.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/schema.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilterWhere.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py From 5eb1c559e5553f3d6bff2ab71bfd7e11e6d4d1da Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 15:51:00 +0800 Subject: [PATCH 101/187] fix:save offset if latest --- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/dnode/vnode/src/tq/tqUtil.c | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 17eac7d096..7557ddb4e9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -377,7 +377,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } taosWUnLockLatch(&pTq->lock); - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosMsleep(10); } @@ -410,7 +410,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { while (tqIsHandleExec(pHandle)) { - tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); + tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); taosMsleep(10); } if (pHandle->pRef) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5eaf7b240b..e73aed8966 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -121,6 +121,12 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); + STqOffset offset = {0}; + strcpy(offset.subKey, pRequest->subKey); + if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + return -1; + } } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); From f984e67eaa33dd4f5e2fb8ce06d4a97c80b0696b Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 29 May 2023 16:04:35 +0800 Subject: [PATCH 102/187] fix(query): invalid buffer page --- source/libs/executor/inc/executil.h | 1 + source/util/src/tpagedbuf.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 0f169b0c2e..05fdba61d7 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -136,6 +136,7 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); if (!bufPage) { + uFatal("failed to get the buffer page:%d since %s", pos->pageId, terrstr()); return NULL; } if (forUpdate) { diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 218a47ea62..2fb189ec9d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -482,7 +482,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); if (*pInfo != *pi) { - terrno = TSDB_CODE_QRY_SYS_ERROR; + terrno = TSDB_CODE_APP_ERROR; uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id); return NULL; } From ccf57129bb6cadda1307b528012ab066ba474eeb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 29 May 2023 16:33:31 +0800 Subject: [PATCH 103/187] fix: show create table privilege issue --- source/libs/catalog/src/ctgUtil.c | 4 ++-- source/libs/parser/src/parAstParser.c | 2 +- source/libs/parser/src/parAuthenticator.c | 2 +- source/libs/parser/src/parUtil.c | 11 +++++++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index ae8fe0cca4..ca63f794e7 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1482,7 +1482,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { switch (pReq->type) { case AUTH_TYPE_READ: { - if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { req->singleType = AUTH_TYPE_READ; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { @@ -1498,7 +1498,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { break; } case AUTH_TYPE_WRITE: { - if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { req->singleType = AUTH_TYPE_WRITE; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5db1f5dbdb..df080b574c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -581,7 +581,7 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_READ, + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, pCxt->pMetaCache); } return code; diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 1586d8128b..251d3bd0cb 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -175,7 +175,7 @@ static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) { static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) { SNode* pTagCond = NULL; // todo check tag condition for subtable - return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_READ, &pTagCond); + return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, &pTagCond); } static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 14da6f8aab..9c95e2b17c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -498,7 +498,7 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, char* pStr) { - return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL != pTable && '\0' == pTable[0]) ? NULL : pTable, + return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, type); } @@ -524,6 +524,9 @@ static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { strncpy(pStr, pStart, p - pStart); *pNext = ++p; } + if (*pStart == '`' && *(pStart + 1) == '`') { + *pStr = 0; + } } static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) { @@ -532,7 +535,11 @@ static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUser pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p); getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p); getStringFromAuthStr(p, pUserAuth->tbName.tname, &p); - pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + if (pUserAuth->tbName.tname[0]) { + pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + } else { + pUserAuth->tbName.type = TSDB_DB_NAME_T; + } pUserAuth->type = getIntegerFromAuthStr(p, &p); } From 9eb15e41efc8a3b936746151163018420bf17b48 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 16:44:03 +0800 Subject: [PATCH 104/187] fix:put poll to push manager if wal not exist when offset is latest --- source/dnode/vnode/src/tq/tqUtil.c | 40 ++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index e73aed8966..d30ed638f1 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -87,12 +87,13 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { return 0; } -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest) { +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, bool* pBlockReturned) { uint64_t consumerId = pRequest->consumerId; STqOffsetVal reqOffset = pRequest->reqOffset; STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); int32_t vgId = TD_VID(pTq->pVnode); + *pBlockReturned = false; // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. if (pOffset != NULL) { *pOffsetVal = pOffset->val; @@ -120,12 +121,27 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); - tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); - STqOffset offset = {0}; - strcpy(offset.subKey, pRequest->subKey); - if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { - terrno = TSDB_CODE_PAR_INTERNAL_ERROR; - return -1; + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); + + tqOffsetResetToLog(&dataRsp.rspOffset, pHandle->pRef->refVer); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, + pHandle->subKey, vgId, dataRsp.rspOffset.version); + int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + tDeleteSMqDataRsp(&dataRsp); + + *pBlockReturned = true; + return code; + } else { + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, pRequest); + tqOffsetResetToLog(&taosxRsp.rspOffset, pHandle->pRef->refVer); + int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); + tDeleteSTaosxRsp(&taosxRsp); + + *pBlockReturned = true; + return code; } } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", @@ -154,8 +170,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST || (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId)) { + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) { // lock taosWLockLatch(&pTq->lock); code = tqRegisterPushHandle(pTq, pHandle, pMsg); @@ -296,11 +311,16 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ // 1. reset the offset if needed if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { // handle the reset offset cases, according to the consumer's choice. - code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest); + bool blockReturned = false; + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); if (code != 0) { return code; } + // empty block returned, quit + if (blockReturned) { + return 0; + } } else { // use the consumer specified offset // the offset value can not be monotonious increase?? offset = reqOffset; From 07bdb95b1a36f8b999d5466dee54d6d21cdc58df Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 17:10:11 +0800 Subject: [PATCH 105/187] fix:compile error --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqMeta.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/executor/src/executil.c | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 5225a7a644..66418e6841 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -82,7 +82,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); -int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* node, SArray **tableList); +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList); /** * set the task Id, usually used by message queue process diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fb9f620c20..01a60a99e4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -713,7 +713,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } SArray* tbUidList = NULL; - ret = qGetTableList(req.suid, pVnode->pMeta, pVnode, pHandle->execHandle.execTb.node, &tbUidList); + ret = qGetTableList(req.suid, pVnode, pHandle->execHandle.execTb.node, &tbUidList); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d vgId:%d, subkey:%s consumer:0x%" PRIx64, ret, pVnode->config.vgId, req.subKey, pHandle->consumerId); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 32f4931fae..d4371f1264 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -347,7 +347,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { } SArray* tbUidList = NULL; - int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList); + int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle.subKey, handle.consumerId); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index fea5104017..a5ec0d4def 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1084,7 +1084,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* list = NULL; - int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list); + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); taosArrayDestroy(list); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index e359cf51e3..19b64d9833 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1152,14 +1152,14 @@ _end: return code; } -int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* node, SArray **tableList){ +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList){ SSubplan *pSubplan = (SSubplan *)node; SScanPhysiNode pNode = {0}; pNode.suid = suid; pNode.uid = suid; pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); - int code = getTableList(metaHandle, pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList"); + int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList"); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From 41ed0138b437630f0b6cd0d08701c1ad0e85463c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 17:25:11 +0800 Subject: [PATCH 106/187] fix:compile error --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 10 +++++----- source/dnode/vnode/src/tq/tqMeta.c | 9 ++++----- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/executor/src/executil.c | 4 ++-- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 66418e6841..e46757f6f8 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -82,7 +82,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); -int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList); +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList, void* pTaskInfo); /** * set the task Id, usually used by message queue process diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 01a60a99e4..f8116f49ef 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -712,8 +712,12 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } } + buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, + (SSnapContext**)(&handle.sContext)); + pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); + SArray* tbUidList = NULL; - ret = qGetTableList(req.suid, pVnode, pHandle->execHandle.execTb.node, &tbUidList); + ret = qGetTableList(req.suid, pVnode, pHandle->execHandle.execTb.node, &tbUidList, pHandle->execHandle.task); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d vgId:%d, subkey:%s consumer:0x%" PRIx64, ret, pVnode->config.vgId, req.subKey, pHandle->consumerId); taosArrayDestroy(tbUidList); @@ -723,10 +727,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); - - buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, - (SSnapContext**)(&handle.sContext)); - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); } taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index d4371f1264..0edffd7f05 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -345,9 +345,12 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { return -1; } } + buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, + handle.fetchMeta, (SSnapContext**)(&reader.sContext)); + handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); SArray* tbUidList = NULL; - int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList); + int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList, handle.execHandle.task); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle.subKey, handle.consumerId); taosArrayDestroy(tbUidList); @@ -357,10 +360,6 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); - - buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, - handle.fetchMeta, (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); taosWLockLatch(&pTq->lock); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index a5ec0d4def..6ef232995a 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1084,7 +1084,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* list = NULL; - int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list); + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list, pTqHandle->execHandle.task); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); taosArrayDestroy(list); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 19b64d9833..a168050645 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1152,14 +1152,14 @@ _end: return code; } -int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList){ +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList, void* pTaskInfo){ SSubplan *pSubplan = (SSubplan *)node; SScanPhysiNode pNode = {0}; pNode.suid = suid; pNode.uid = suid; pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); - int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList"); + int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From 16e14b1c828a75a6ad739b41d601b0dd34acb795 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 17:27:35 +0800 Subject: [PATCH 107/187] fix:compile error --- source/libs/executor/src/executil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a168050645..f4531729c4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1159,7 +1159,8 @@ int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList pNode.uid = suid; pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); - int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); + uint8_t digest[17] = {0}; + int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, digest, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From 88350280020e952e7c1d7a04aa044943da6355c8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 10:21:18 +0800 Subject: [PATCH 108/187] fix:deal with node is null --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c7677878bf..3752a56b92 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1160,7 +1160,7 @@ int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); uint8_t digest[17] = {0}; - int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, digest, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); + int code = getTableList(pVnode, &pNode, pSubplan ? pSubplan->pTagCond : NULL, pSubplan ? pSubplan->pTagIndexCond : NULL, pTableListInfo, digest, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From 5d2c78ba6b5d37e3224d2d3f6587de43cdec23bd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 10:39:00 +0800 Subject: [PATCH 109/187] fix:memory leak --- source/libs/nodes/src/nodesUtilFuncs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 17a5677987..70fc45e07a 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -921,6 +921,7 @@ void nodesDestroyNode(SNode* pNode) { break; case QUERY_NODE_CREATE_TOPIC_STMT: nodesDestroyNode(((SCreateTopicStmt*)pNode)->pQuery); + nodesDestroyNode(((SCreateTopicStmt*)pNode)->pWhere); break; case QUERY_NODE_DROP_TOPIC_STMT: // no pointer field case QUERY_NODE_DROP_CGROUP_STMT: // no pointer field From 757ac9b1d9749be7a2d52de06bcc67e4b7af8419 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 14:20:21 +0800 Subject: [PATCH 110/187] fix:add test cases --- tests/system-test/7-tmq/stbFilterWhere.py | 72 ++++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/tests/system-test/7-tmq/stbFilterWhere.py b/tests/system-test/7-tmq/stbFilterWhere.py index baf042d71c..8d8d046cef 100644 --- a/tests/system-test/7-tmq/stbFilterWhere.py +++ b/tests/system-test/7-tmq/stbFilterWhere.py @@ -56,7 +56,7 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) return - def tmqCaseError(self, topicName, condition): + def tmqCase_columnError(self, topicName, condition): tdLog.printNoPrefix("======== test case error: ") paraDict = {'dbName': 'dbt', 'dropFlag': 1, @@ -140,14 +140,82 @@ class TDTestCase: tdLog.printNoPrefix("======== test case end ...... ") + def tmqCase_addNewTable_dropTag(self, topicName, condition): + tdLog.printNoPrefix("======== test case1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with tag filter") + topicString = "create topic %s as stable %s.%s where %s" %(topicName, paraDict['dbName'], paraDict['stbName'], condition) + tdLog.info("create topic sql: %s"%topicString) + tdSql.execute(topicString) + + queryString = "select * from %s.%s where %s" %(paraDict['dbName'], paraDict['stbName'], condition) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows() + 1) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicName + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + #add new table with one data + tdLog.info("start insert data") + insertString = "insert into %s.tmp using %s.%s tags(1, 1, 1, 't4', 't5') values(now, 1, 1, 1, 'c4', 'c5', now)" %(paraDict['dbName'], paraDict['dbName'], paraDict['stbName']) + tdSql.execute(insertString) + + #test drop tag + tdSql.error("alter stable %s.%s drop tag t1" %(paraDict['dbName'], paraDict['stbName'])) + tdSql.execute("alter stable %s.%s drop tag t2" %(paraDict['dbName'], paraDict['stbName'])) + tdSql.execute("alter stable %s.%s drop column c2" %(paraDict['dbName'], paraDict['stbName'])) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + tdLog.printNoPrefix("======== test case1 end ...... ") + def run(self): tdSql.prepare() self.prepareTestEnv() - self.tmqCaseError("t1", "c1 = 4 and t1 = 3") + self.tmqCase_columnError("t1", "c1 = 4 and t1 = 3") self.tmqCase("t2", "2 > 1") self.tmqCase("t3", "t4 = 'beijing'") self.tmqCase("t4", "t4 > t3") self.tmqCase("t5", "t3 = t4") + self.tmqCase_addNewTable_dropTag("t6", "t1 = 1") def stop(self): tdSql.close() From 6a7d8858dc516c3b1fe79ea2194e5159871ac99e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 14:36:51 +0800 Subject: [PATCH 111/187] test: add smaBasic.py test case --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/smaBasic.py | 297 ++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 tests/system-test/2-query/smaBasic.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcfb392bb4..f0543f2456 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -294,6 +294,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaBasic.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py new file mode 100644 index 0000000000..917a2d3a08 --- /dev/null +++ b/tests/system-test/2-query/smaBasic.py @@ -0,0 +1,297 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + self.ts += 1 + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null)" + else: + value += f"{c2})" + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1600000000000 + self.childCnt = 5 + self.childRow = 2000000 + self.batchSize = 2000 + self.c1_values = [] + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 5 replica 3" + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # insert data + self.insertData() + + # query sql value + def queryValue(self, sql): + tdSql.query(sql) + return tdSql.getData(0, 0) + + # sum + def checkCorrentSum(self): + # query count + sql = "select sum(c1) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c1Cnt: + tdLog.exit(f"Sum Not Expect. expect={self.c1Cnt} query={val} sql:{sql}") + return + + # not + sql1 = "select sum(c1) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + # over than + sql1 = "select sum(c1) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + tdLog.info(f"check correct sum on c1 successfully.") + + # check result + def checkResult(self, fun, val, val1, val2, sql1, sql2): + if fun == "count": + if val != val1 + val2: + tdLog.exit(f"{fun} NOT SAME. val != val1 + val2. val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "max": + if val != max([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != max(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "min": + if val != min([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != min(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + + # sum + def checkCorrentFun(self, fun, expectVal): + # query + sql = f"select {fun}(c2) from st" + val = self.queryValue(sql) + if val != expectVal: + tdLog.exit(f"{fun} Not Expect. expect={expectVal} query={val} sql:{sql}") + return + + # not + sql1 = f"select {fun}(c2) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # over than + sql1 = f"select {fun}(c2) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # successful + tdLog.info(f"check correct {fun} on c2 successfully.") + + # check query corrent + def checkCorrect(self): + # count + self.checkCorrentFun("count", self.c2Cnt) + # max + self.checkCorrentFun("max", self.c2Max) + # min + self.checkCorrentFun("min", self.c2Min) + # sum + self.checkCorrentSum() + + # c2 sum + sql = "select sum(c2) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c2Sum: + tdLog.exit(f"c2 Sum Not Expect. expect={self.c2Sum} query={val} sql:{sql}") + return + + def checkPerformance(self): + # have sma caculate + sql1 = "select count(*) from st" + stime = time.time() + tdSql.execute(sql1, 1) + spend1 = time.time() - stime + + + # no sma caculate + sql2 = "select count(*) from st where c2 != 8764231 or c2 is null" + stime = time.time() + tdSql.execute(sql2, 1) + spend2 = time.time() - stime + + time1 = "%.2f"%(spend1*1000) + time2 = "%.2f"%(spend2*1000) + if spend2 < spend1 * 5: + tdLog.exit(f"performance not passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + return + tdLog.info(f"performance passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # query + self.checkCorrect() + + # performance + self.checkPerformance() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From ed9eaf541d1d95afac64c18898d2f384a0308031 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 14:37:46 +0800 Subject: [PATCH 112/187] test: add smaBasic.py test case --- tests/system-test/2-query/smaBasic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py index 917a2d3a08..26adc03d2b 100644 --- a/tests/system-test/2-query/smaBasic.py +++ b/tests/system-test/2-query/smaBasic.py @@ -117,7 +117,7 @@ class TDTestCase: self.ts = 1600000000000 self.childCnt = 5 self.childRow = 2000000 - self.batchSize = 2000 + self.batchSize = 5000 self.c1_values = [] # total From 7d699ae579088ce2f075417a6925a0a3eae260e8 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Tue, 30 May 2023 15:16:11 +0800 Subject: [PATCH 113/187] fix sliding window issue --- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 11 ++++++----- source/libs/stream/src/streamState.c | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2d2f377041..f11b083ad6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1883,7 +1883,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pInfo->pRecoverRes != NULL) { pInfo->blockRecoverContiCnt++; calBlockTbName(pInfo, pInfo->pRecoverRes); - if (pInfo->pUpdateInfo) { + if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) { TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index aa0aa9799b..2cdf9bb15c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1623,7 +1623,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; - if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { + if (!pScanInfo->pUpdateInfo) { pScanInfo->pUpdateInfo = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark); } @@ -2150,28 +2150,29 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB } void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SInterval* pInterval) { - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); TSKEY* tsData = (TSKEY*)pStartCol->pData; - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); TSKEY* tsEndData = (TSKEY*)pEndCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; int32_t chId = getChildIndex(pBlock); for (int32_t i = 0; i < pBlock->info.rows; i++) { TSKEY winTs = tsData[i]; - while (winTs < tsEndData[i]) { + while (winTs <= tsEndData[i]) { SWinKey winRes = {.ts = winTs, .groupId = groupIdData[i]}; void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey)); if (chIds) { SArray* chArray = *(SArray**)chIds; int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); if (index != -1) { - qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId); + qDebug("===stream===retrive window %" PRId64 " delete child id %d", winRes.ts, chId); taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); + qDebug("===stream===retrive pull data over.window %" PRId64 , winRes.ts); } } } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ba24a581e9..71a21ac150 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -1062,7 +1062,7 @@ _end: } int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { - qWarn("try to write to cf parname"); + qDebug("try to write to cf parname"); #ifdef USE_ROCKSDB if (tSimpleHashGetSize(pState->parNameMap) > MAX_TABLE_NAME_NUM) { if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) { From de7231e59e839c8d6fc58439bc0bdf2b2c805e75 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 15:35:59 +0800 Subject: [PATCH 114/187] fix(stream): fix memory leak in handling dispatch msg when output buffer is full. --- source/libs/stream/src/stream.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 8cc1ef1dd3..d3e4b23ad1 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -129,11 +129,11 @@ int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pR qDebug("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId, pTask->id.idStr); } else { - if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock) == 0) { - status = TASK_INPUT_STATUS__NORMAL; - } else { // input queue is full, upstream is blocked now - status = TASK_INPUT_STATUS__BLOCKED; - } + int32_t code = tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock); + // input queue is full, upstream is blocked now + status = (code == TSDB_CODE_SUCCESS)? TASK_INPUT_STATUS__NORMAL:TASK_INPUT_STATUS__BLOCKED; + + } // rsp by input status @@ -303,6 +303,7 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { taosFreeQitem(pItem); return -1; } + taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { @@ -310,6 +311,8 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); + destroyStreamDataBlock((SStreamDataBlock*) pItem); + taosFreeQitem(pItem); return -1; } From 772ef3bd19b5264d54766bb7397b7c135d741cde Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 15:39:15 +0800 Subject: [PATCH 115/187] test: modfiy insert rows to small --- tests/system-test/2-query/smaBasic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py index 26adc03d2b..43c379ee53 100644 --- a/tests/system-test/2-query/smaBasic.py +++ b/tests/system-test/2-query/smaBasic.py @@ -116,9 +116,8 @@ class TDTestCase: # init self.ts = 1600000000000 self.childCnt = 5 - self.childRow = 2000000 + self.childRow = 1000000 self.batchSize = 5000 - self.c1_values = [] # total self.c1Cnt = 0 @@ -262,7 +261,7 @@ class TDTestCase: time1 = "%.2f"%(spend1*1000) time2 = "%.2f"%(spend2*1000) - if spend2 < spend1 * 5: + if spend2 < spend1 * 8: tdLog.exit(f"performance not passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") return tdLog.info(f"performance passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") From b8a4df414699a94e0c4a2c775d101ae8df3c6123 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 16:50:20 +0800 Subject: [PATCH 116/187] fix:avoid put tag to select if where condition is always true or false --- source/libs/parser/src/parTranslater.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 102b7479be..c10ee5d988 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5899,15 +5899,15 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* nodesDestroyList(colCxt.pTags); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Columns are forbidden in where clause"); } - if (NULL == colCxt.pTags) { - for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) { - SSchema* tag = &pMeta->schema[pMeta->tableInfo.numOfColumns + i]; + if (NULL == colCxt.pTags) { // put one column to select +// for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { + SSchema* column = &pMeta->schema[0]; SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - strcpy(col->colName, tag->name); + strcpy(col->colName, column->name); strcpy(col->node.aliasName, col->colName); strcpy(col->node.userAlias, col->colName); addTagList(&colCxt.pTags, (SNode*)col); - } +// } } *ppProjection = colCxt.pTags; From fcdf25793b5890b7f39d2f47ab1b004a43194877 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 16:52:53 +0800 Subject: [PATCH 117/187] fix:remove useless code --- tests/script/tsim/tmq/topic.sim | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index 85a23055a3..073c8c2532 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -108,7 +108,6 @@ if $rows != 6 then return -1 endi -return -1 sql create topic topic_stable_1 as stable stb where t1 > 0 sql create topic topic_stable_1 as stable stb where t1 > 0 and t1 < 0 sql create topic topic_stable_1 as stable stb where 1 > 0 From e96ed81ec0df8c6eacdaa0c38c44e8099c9c3346 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 17:29:01 +0800 Subject: [PATCH 118/187] fix(stream): add input queue capacity check. --- source/libs/stream/src/stream.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index d3e4b23ad1..0d772247b4 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -132,8 +132,6 @@ int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pR int32_t code = tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock); // input queue is full, upstream is blocked now status = (code == TSDB_CODE_SUCCESS)? TASK_INPUT_STATUS__NORMAL:TASK_INPUT_STATUS__BLOCKED; - - } // rsp by input status @@ -304,10 +302,15 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { return -1; } - taosWriteQitem(pTask->inputQueue->queue, pItem); + int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); + if (code != TSDB_CODE_SUCCESS) { + streamDataSubmitDestroy(px); + taosFreeQitem(pItem); + return code; + } } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { - if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && (tInputQueueIsFull(pTask))) { + if (/*(pTask->taskLevel == TASK_LEVEL__SOURCE) && */(tInputQueueIsFull(pTask))) { qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); @@ -317,10 +320,16 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { } qDebug("s-task:%s data block enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size); - taosWriteQitem(pTask->inputQueue->queue, pItem); + int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamDataBlock((SStreamDataBlock*) pItem); + taosFreeQitem(pItem); + return code; + } } else if (type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (type == STREAM_INPUT__GET_RES) { + // use the default memory limit, refactor later. taosWriteQitem(pTask->inputQueue->queue, pItem); qDebug("s-task:%s data res enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size); } From 937f67f07b0a89dc1df5a5e01c4dbfa2e31b76e5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 18:02:42 +0800 Subject: [PATCH 119/187] fix(stream): fix invalid free. --- source/libs/stream/src/stream.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 0d772247b4..03013dbf3e 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -315,7 +315,6 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); destroyStreamDataBlock((SStreamDataBlock*) pItem); - taosFreeQitem(pItem); return -1; } From d9cc9f21574f4799e827c5792e85e4efa4ce3db1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 18:03:21 +0800 Subject: [PATCH 120/187] fix(stream): remove invalid free. --- source/libs/stream/src/stream.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 03013dbf3e..6b69f19427 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -322,7 +322,6 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); if (code != TSDB_CODE_SUCCESS) { destroyStreamDataBlock((SStreamDataBlock*) pItem); - taosFreeQitem(pItem); return code; } } else if (type == STREAM_INPUT__CHECKPOINT) { From fc20ca8423dbe4b637d3465adea0f8fce692072e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 18:42:23 +0800 Subject: [PATCH 121/187] fix(stream): set the correct rps flag as the dispatch rsp. --- source/libs/stream/src/stream.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 6b69f19427..8bca0e6ec2 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -233,8 +233,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { - ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED); - qDebug("s-task:%s receive dispatch rsp, code: %x", pTask->id.idStr, code); + qDebug("s-task:%s receive dispatch rsp, status:%d code:%d", pTask->id.idStr, pRsp->inputStatus, code); if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); @@ -246,13 +245,16 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); ASSERT(old == TASK_OUTPUT_STATUS__WAIT); + + // the input queue of the (down stream) task that receive the output data is full, so the TASK_INPUT_STATUS_BLOCKED is rsp + // todo we need to send EMPTY PACKAGE to detect if the input queue is available for output of upstream task, every 50 ms. if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { // TODO: init recover timer - ASSERT(0); + qError("s-task:%s inputQ of downstream task:0x%x is full, need to block output", pTask->id.idStr, pRsp->downstreamTaskId); return 0; } - // continue dispatch one block to down stream in pipeline + // otherwise, continue dispatch the first block to down stream task in pipeline streamDispatchStreamBlock(pTask); return 0; } From 67becbd4d1562dba82fcbb7b31a16bd7a0c4446a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 31 May 2023 01:10:36 +0000 Subject: [PATCH 122/187] change db param --- source/libs/stream/src/streamBackendRocksdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f7638a42ae..f7a8ec4564 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -92,6 +92,7 @@ void* streamBackendInit(const char* path) { rocksdb_options_set_recycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 2); rocksdb_options_set_info_log_level(opts, 0); + rocksdb_options_set_db_write_buffer_size(opts, 256 << 20); pHandle->env = env; pHandle->dbOpt = opts; From 23bcfc2fd0714f9ef549c638635fc26ec7e31f2d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 31 May 2023 09:16:47 +0800 Subject: [PATCH 123/187] enh: stmt column length validation --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 5 +- source/libs/parser/src/parInsertStmt.c | 7 +- tests/script/api/batchprepare.c | 93 +++++++++++++++++++------- 4 files changed, 79 insertions(+), 28 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 8be5cb4d41..e04bdd1b07 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -145,7 +145,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); // for stmt bind -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen); void tColDataSortMerge(SArray *colDataArr); // for raw block diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 55204045ba..688f3b006c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2503,7 +2503,7 @@ _exit: return code; } -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen) { int32_t code = 0; if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { @@ -2515,6 +2515,9 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { if (pBind->is_null && pBind->is_null[i]) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; + } else if (pBind->length[i] > buffMaxLen) { + uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen); + return TSDB_CODE_INVALID_PARA; } else { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 922a0f45ff..8284913975 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -266,7 +266,10 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in pBind = bind + c; } - tColDataAddValueByBind(pCol, pBind); + code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); + if (code) { + goto _return; + } } qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); @@ -309,7 +312,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu pBind = bind; } - tColDataAddValueByBind(pCol, pBind); + tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); qDebug("stmt col %d bind %d rows data", colIdx, rowNum); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 99507ef5c3..80bf5b90af 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -16,8 +16,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; -int32_t optrIdxList[] = {0, 7}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR}; +int32_t optrIdxList[] = {5, 11}; typedef struct { char* oper; @@ -123,6 +123,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos); +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos); enum { TTYPE_INSERT = 1, @@ -190,6 +191,7 @@ CaseCfg gCase[] = { {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:NG-TBNEXISTS",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, false, insertNonExistsTb, 10, 10, 1, 3, 0, 0, 1, -1}, + {"query:NG-VARLENERR",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, true, insertVarLenErr, 10, 10, 1, 3, 0, 0, 1, -1}, // {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, // {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, @@ -319,7 +321,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper #if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper - .bindNullNum = 1, + .bindNullNum = 0, .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, @@ -329,18 +331,19 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindTagNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, - .bindColTypeList = NULL, + .bindColTypeList = bindColTypeList, .optrIdxListNum = 0, - .optrIdxList = NULL, + .optrIdxList = optrIdxList, .checkParamNum = false, .printRes = true, .runTimes = 0, .caseRunIdx = -1, - //.optrIdxListNum = tListLen(optrIdxList), - //.optrIdxList = optrIdxList, - //.bindColTypeNum = tListLen(bindColTypeList), - //.bindColTypeList = bindColTypeList, - .caseIdx = 8, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseRunIdx = -1, + .caseIdx = 24, .caseNum = 1, .caseRunNum = 1, }; @@ -1439,14 +1442,17 @@ void bpShowBindParam(TAOS_MULTI_BIND *bind, int32_t num) { } } -int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { +int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, bool expectFail) { static int32_t n = 0; - bpCheckColFields(stmt, bind); + if (!expectFail) { + bpCheckColFields(stmt, bind); + } if (gCurCase->bindRowNum > 1) { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1454,6 +1460,7 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { if (taos_stmt_bind_single_param_batch(stmt, bind+i, i)) { + if (expectFail) continue; printf("!!!taos_stmt_bind_single_param_batch %d error:%s\n", taos_stmt_errstr(stmt), i); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1463,12 +1470,14 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); } } else { if (taos_stmt_bind_param(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1531,7 +1540,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1583,7 +1592,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1641,7 +1650,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1691,7 +1700,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1759,7 +1768,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1811,7 +1820,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1872,7 +1881,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1938,7 +1947,7 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2005,7 +2014,7 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos) { if (gCaseCtrl.checkParamNum) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2065,7 +2074,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2119,7 +2128,7 @@ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2167,7 +2176,7 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2234,6 +2243,42 @@ int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) { return 0; } +void bpAddWrongVarBuffLen(TAOS_MULTI_BIND* pBind) { + for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { + if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) { + *pBind[i].length += 100; + } + } +} + +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + code = bpSetTableNameTags(&data, 0, "t0", stmt); + if (code != 0){ + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpAddWrongVarBuffLen(data.pBind); + + if (bpBindParam(stmt, data.pBind, true)) { + exit(1); + } + + destroyData(&data); + + return 0; +} int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { From e4866d1803c6987a377ab3eb8dfd96a9cdbe2422 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 31 May 2023 10:38:22 +0800 Subject: [PATCH 124/187] fix:ci test cases error --- tests/script/tsim/tmq/topic.sim | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index 073c8c2532..78c4c561af 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -109,12 +109,14 @@ if $rows != 6 then endi sql create topic topic_stable_1 as stable stb where t1 > 0 -sql create topic topic_stable_1 as stable stb where t1 > 0 and t1 < 0 -sql create topic topic_stable_1 as stable stb where 1 > 0 -sql create topic topic_stable_1 as stable stb where last(t1) > 0 -sql create topic topic_stable_1 as stable stb where tbname is not null -sql create topic topic_stable_1 as stable stb where tbname > 'a' -sql create topic topic_stable_1 as stable stb where tbname > 0 and xx < 0 -sql create topic topic_stable_1 as stable stb where tbname > 0 and c1 < 0 +sql create topic topic_stable_2 as stable stb where t1 > 0 and t1 < 0 +sql create topic topic_stable_3 as stable stb where 1 > 0 +sql create topic topic_stable_4 as stable stb where abs(t1) > 0 +sql_error create topic topic_stable_5 as stable stb where last(t1) > 0 +sql_error create topic topic_stable_5 as stable stb where sum(t1) > 0 +sql create topic topic_stable_6 as stable stb where tbname is not null +sql create topic topic_stable_7 as stable stb where tbname > 'a' +sql_error create topic topic_stable_8 as stable stb where tbname > 0 and xx < 0 +sql_error create topic topic_stable_9 as stable stb where tbname > 0 and c1 < 0 system sh/exec.sh -n dnode1 -s stop -x SIGINT From 3b0a6d7f0b0e5082ef074c536b0afd45dff3c36a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 31 May 2023 03:15:25 +0000 Subject: [PATCH 125/187] change db param --- include/common/tglobal.h | 2 +- source/common/src/tglobal.c | 16 ++++++++++------ source/libs/stream/src/streamBackendRocksdb.c | 12 +++++++++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 157e37f080..ced94e0f11 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -29,7 +29,6 @@ extern "C" { #define SLOW_LOG_TYPE_OTHERS 0x4 #define SLOW_LOG_TYPE_ALL 0xFFFFFFFF - // cluster extern char tsFirst[]; extern char tsSecond[]; @@ -181,6 +180,7 @@ extern bool tsDisableStream; extern int64_t tsStreamBufferSize; extern int64_t tsCheckpointInterval; extern bool tsFilterScalarMode; +extern int32_t tsMaxStreamBackendCache; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c648f8551a..cc368167f8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -60,6 +60,7 @@ int32_t tsNumOfQnodeQueryThreads = 4; int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeStreamThreads = 4; int32_t tsNumOfSnodeWriteThreads = 1; +int32_t tsMaxStreamBackendCache = 256; // M // sync raft int32_t tsElectInterval = 25 * 1000; @@ -105,7 +106,7 @@ int32_t tsQueryPolicy = 1; int32_t tsQueryRspPolicy = 0; int64_t tsQueryMaxConcurrentTables = 200; // unit is TSDB_TABLE_NUM_UNIT bool tsEnableQueryHb = false; -bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true +bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -117,8 +118,8 @@ int32_t tsRedirectFactor = 2; int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; -int32_t tsMetaCacheMaxSize = -1; // MB -int32_t tsSlowLogThreshold = 3; // seconds +int32_t tsMetaCacheMaxSize = -1; // MB +int32_t tsSlowLogThreshold = 3; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; /* @@ -349,7 +350,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; - if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) + return -1; if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, 1) != 0) return -1; if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, true) != 0) return -1; if (cfgAddString(pCfg, "slowLogScope", "", true) != 0) return -1; @@ -524,6 +526,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1; if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, 0) != 0) return -1; GRANT_CFG_ADD; return 0; @@ -781,7 +784,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; - tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval; + tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval; tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; @@ -902,7 +905,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i64; tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval; - + tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32; + GRANT_CFG_GET; return 0; } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f7a8ec4564..c8a6597bad 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -38,6 +38,15 @@ typedef struct { rocksdb_comparator_t** pCompares; } RocksdbCfInst; +uint32_t nextPow2(uint32_t x) { + x = x - 1; + x = x | (x >> 1); + x = x | (x >> 2); + x = x | (x >> 4); + x = x | (x >> 8); + x = x | (x >> 16); + return x + 1; +} int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); void destroyRocksdbCfInst(RocksdbCfInst* inst); @@ -92,7 +101,8 @@ void* streamBackendInit(const char* path) { rocksdb_options_set_recycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 2); rocksdb_options_set_info_log_level(opts, 0); - rocksdb_options_set_db_write_buffer_size(opts, 256 << 20); + uint32_t dbLimit = nextPow2(tsMaxStreamBackendCache); + rocksdb_options_set_db_write_buffer_size(opts, dbLimit << 20); pHandle->env = env; pHandle->dbOpt = opts; From ed851b522cc6a9da43a009cb09145bc05fd4183a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 31 May 2023 03:30:10 +0000 Subject: [PATCH 126/187] change db param --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cc368167f8..c5646d92d1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -60,7 +60,7 @@ int32_t tsNumOfQnodeQueryThreads = 4; int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeStreamThreads = 4; int32_t tsNumOfSnodeWriteThreads = 1; -int32_t tsMaxStreamBackendCache = 256; // M +int32_t tsMaxStreamBackendCache = 128; // M // sync raft int32_t tsElectInterval = 25 * 1000; From 18b971c666873525f0f8fade035af51f5995d7eb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 17:06:38 +0800 Subject: [PATCH 127/187] fix(query): check the version range when dump partial rows of file block to sdata block directly. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 54 ++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 362934ec84..9ab8654bd9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1121,6 +1121,27 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, pReader->order); } + if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer)|| + (pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.minVer > pBlock->minVer)) { + int32_t i = endPos; + + if (asc) { + for(; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->verRange.maxVer) { + break; + } + } + } else { + for(; i < pBlock->nRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->verRange.minVer) { + break; + } + } + } + + endPos = i; + } + return endPos; } @@ -1260,10 +1281,11 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { return 0; } + // row index of dump info remain the initial position, let's find the appropriate start position. if ((pDumpInfo->rowIndex == 0 && asc) || (pDumpInfo->rowIndex == pBlock->nRow - 1 && (!asc))) { - if (asc && pReader->window.skey <= pBlock->minKey.ts) { + if (asc && pReader->window.skey <= pBlock->minKey.ts && pReader->verRange.minVer <= pBlock->minVer) { // pDumpInfo->rowIndex = 0; - } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) { + } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts && pReader->verRange.maxVer >= pBlock->maxVer) { // pDumpInfo->rowIndex = pBlock->nRow - 1; } else { // find the appropriate the start position in current block, and set it to be the current rowIndex int32_t pos = asc ? pBlock->nRow - 1 : 0; @@ -1279,6 +1301,29 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { pBlock->maxVer, pReader->idStr); return TSDB_CODE_INVALID_PARA; } + + ASSERT(pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.maxVer >= pBlock->minVer); + + // find the appropriate start position that satisfies the version requirement. + if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer)|| + (pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.minVer > pBlock->minVer)) { + int32_t i = pDumpInfo->rowIndex; + if (asc) { + for(; i < pBlock->nRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->verRange.minVer) { + break; + } + } + } else { + for(; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->verRange.maxVer) { + break; + } + } + } + + pDumpInfo->rowIndex = i; + } } } @@ -1293,6 +1338,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); if (dumpedRows > pReader->resBlockInfo.capacity) { // output buffer check dumpedRows = pReader->resBlockInfo.capacity; + } else if (dumpedRows <= 0) { // no qualified rows in current data block, abort directly. + setBlockAllDumped(pDumpInfo, pReader->window.ekey, pReader->order); + return TSDB_CODE_SUCCESS; } int32_t i = 0; @@ -1848,7 +1896,7 @@ static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBloc SDataBlockToLoadInfo info = {0}; getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader); bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || - info.overlapWithDelInfo || info.overlapWithLastBlock || info.partiallyRequired); + info.overlapWithDelInfo || info.overlapWithLastBlock); return isCleanFileBlock; } From fa5c024b3ee97914ae58e58f5507719eddf19977 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 17:22:59 +0800 Subject: [PATCH 128/187] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9ab8654bd9..7af0936bb5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2857,7 +2857,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // it is a clean block, load it directly if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && pBlock->nRow <= pReader->resBlockInfo.capacity) { - if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { + if (asc || (!hasDataInLastBlock(pLastBlockReader))) { code = copyBlockDataToSDataBlock(pReader); if (code) { goto _end; @@ -2876,7 +2876,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } - SBlockData* pBlockData = &pReader->status.fileBlockData; + SBlockData* pBlockData = &pReader->status.fileBlockData; while (1) { bool hasBlockData = false; @@ -2890,7 +2890,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pDumpInfo->rowIndex += step; - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + pBlock = getCurrentBlock(&pReader->status.blockIter); if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); // NOTE: get the new block info @@ -2918,7 +2918,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + pBlock = getCurrentBlock(&pReader->status.blockIter); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; } From 65b81a803f7edda2ab3b0aa50c08c5c258c6c521 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 18:05:53 +0800 Subject: [PATCH 129/187] refactor: do some internal refactor and add some logs. --- source/libs/stream/src/stream.c | 4 +++- source/libs/stream/src/streamDispatch.c | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 8bca0e6ec2..72fd520498 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -237,7 +237,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); - qDebug("task %d is shuffle, left waiting rsp %d", pTask->id.taskId, leftRsp); + qDebug("s-task:%s is shuffle, left waiting rsp %d", pTask->id.idStr, leftRsp); if (leftRsp > 0) { return 0; } @@ -246,6 +246,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); ASSERT(old == TASK_OUTPUT_STATUS__WAIT); + qDebug("s-task:%s receive dispatch rsp, output status:%d", pTask->id.idStr, pTask->outputStatus); + // the input queue of the (down stream) task that receive the output data is full, so the TASK_INPUT_STATUS_BLOCKED is rsp // todo we need to send EMPTY PACKAGE to detect if the input queue is available for output of upstream task, every 50 ms. if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 401a8b9e74..1e939cb071 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -510,14 +510,17 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { int8_t old = atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); if (old != TASK_OUTPUT_STATUS__NORMAL) { - qDebug("s-task:%s task wait for dispatch rsp, not dispatch now", pTask->id.idStr); + qDebug("s-task:%s task wait for dispatch rsp, not dispatch now, output status:%d", pTask->id.idStr, old); return 0; } + qDebug("s-task:%s start to dispatch msg, output status:%d", pTask->id.idStr, pTask->outputStatus); + SStreamDataBlock* pDispatchedBlock = streamQueueNextItem(pTask->outputQueue); if (pDispatchedBlock == NULL) { - qDebug("s-task:%s stop dispatching since no output in output queue", pTask->id.idStr); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); + qDebug("s-task:%s stop dispatching since no output in output queue, output status:%d", pTask->id.idStr, + pTask->outputStatus); return 0; } @@ -527,6 +530,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { streamQueueProcessFail(pTask->outputQueue); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); + qDebug("s-task:%s failed to dispatch msg to downstream, output status:%d", pTask->id.idStr, pTask->outputStatus); } // this block can be freed only when it has been pushed to down stream. From a3abe25539a07ca87fb051cdbf5fe4056aae8440 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:03:37 +0800 Subject: [PATCH 130/187] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 6 ++++-- source/dnode/vnode/src/tq/tqSink.c | 12 +++++------- source/libs/stream/src/stream.c | 10 ++++++---- source/libs/stream/src/streamExec.c | 2 +- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index dcfc578ac7..917f06ae14 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1272,13 +1272,15 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = ntohl(pRsp->upstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - tqDebug("recv dispatch rsp, code:%x", pMsg->code); + + int32_t vgId = pTq->pStreamMeta->vgId; + tqDebug("vgId:%d recv dispatch rsp, code:%d", vgId, pMsg->code); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } else { - return -1; + tqDebug("vgId:%d failed to find task:0x%x", vgId, taskId); } } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 0bd7d9a57b..6e02d5e21b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -137,7 +137,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d int32_t blockSz = taosArrayGetSize(pBlocks); - tqDebug("vgId:%d, s-task:%s write results blocks:%d into table", TD_VID(pVnode), pTask->id.idStr, blockSz); + tqDebug("vgId:%d, s-task:%s write results %d blocks into table", TD_VID(pVnode), pTask->id.idStr, blockSz); void* pBuf = NULL; SArray* tagArray = NULL; @@ -482,17 +482,15 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d tEncoderClear(&encoder); tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); - SRpcMsg msg = { - .msgType = TDMT_VND_SUBMIT, - .pCont = pBuf, - .contLen = len, - }; - + SRpcMsg msg = { .msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { tqDebug("failed to put into write-queue since %s", terrstr()); } } } + + tqDebug("vgId:%d, s-task:%s write results completed", TD_VID(pVnode), pTask->id.idStr); + _end: taosArrayDestroy(tagArray); taosArrayDestroy(pVals); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 72fd520498..acc69c5a2b 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -126,7 +126,7 @@ int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pR if (pBlock == NULL) { streamTaskInputFail(pTask); status = TASK_INPUT_STATUS__FAILED; - qDebug("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId, + qError("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId, pTask->id.idStr); } else { int32_t code = tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock); @@ -233,7 +233,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { - qDebug("s-task:%s receive dispatch rsp, status:%d code:%d", pTask->id.idStr, pRsp->inputStatus, code); + qDebug("s-task:%s receive dispatch rsp, output status:%d code:%d", pTask->id.idStr, pRsp->inputStatus, code); if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); @@ -246,13 +246,15 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); ASSERT(old == TASK_OUTPUT_STATUS__WAIT); - qDebug("s-task:%s receive dispatch rsp, output status:%d", pTask->id.idStr, pTask->outputStatus); - // the input queue of the (down stream) task that receive the output data is full, so the TASK_INPUT_STATUS_BLOCKED is rsp // todo we need to send EMPTY PACKAGE to detect if the input queue is available for output of upstream task, every 50 ms. if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { // TODO: init recover timer qError("s-task:%s inputQ of downstream task:0x%x is full, need to block output", pTask->id.idStr, pRsp->downstreamTaskId); + + atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); + qError("s-task:%s ignore error, and reset task output status:%d", pTask->id.idStr, pTask->outputStatus); + return 0; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 95b97e080a..d6f5533db4 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -385,7 +385,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { if (pTask->taskLevel == TASK_LEVEL__SINK) { ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); - qDebug("s-task:%s sink node start to sink result. numOfBlocks:%d", pTask->id.idStr, batchSize); + qDebug("s-task:%s sink task start to sink %d blocks", pTask->id.idStr, batchSize); streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pInput); continue; } From 9e021087a935c79bf56bef5e498e84427a7cdb45 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:04:53 +0800 Subject: [PATCH 131/187] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 917f06ae14..6ec9020759 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1281,6 +1281,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { return 0; } else { tqDebug("vgId:%d failed to find task:0x%x", vgId, taskId); + return TSDB_CODE_INVALID_MSG; } } From 04f3784696cc11f534451196b2aee4a9f2ea9d98 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 31 May 2023 11:05:03 +0000 Subject: [PATCH 132/187] change log level --- source/libs/stream/src/streamBackendRocksdb.c | 14 +++++++------- source/libs/stream/src/streamDispatch.c | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c8a6597bad..9d94a28c8a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1026,10 +1026,10 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ if (err != NULL) { \ taosMemoryFree(err); \ - qDebug("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ + qError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ code = -1; \ } else { \ - qDebug("streamState str:%s succ to write to %s, valLen:%d", toString, funcname, vLen); \ + qTrace("streamState str:%s succ to write to %s, valLen:%d", toString, funcname, vLen); \ } \ taosMemoryFree(ttlV); \ } while (0); @@ -1056,10 +1056,10 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ if (val == NULL) { \ if (err == NULL) { \ - qDebug("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ + qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ funcname); \ } else { \ - qDebug("streamState str: %s failed to read from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ + qError("streamState str: %s failed to read from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ err); \ taosMemoryFreeClear(err); \ } \ @@ -1068,11 +1068,11 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa char* p = NULL; \ int32_t len = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ if (len < 0) { \ - qDebug("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ + qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ funcname); \ code = -1; \ } else { \ - qDebug("streamState str: %s succ to read from %s_%s, valLen:%d", toString, pState->pTdbState->idstr, funcname, \ + qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, pState->pTdbState->idstr, funcname, \ len); \ } \ taosMemoryFree(val); \ @@ -1107,7 +1107,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa taosMemoryFree(err); \ code = -1; \ } else { \ - qDebug("streamState str: %s succ to del from %s_%s", toString, pState->pTdbState->idstr, funcname); \ + qTrace("streamState str: %s succ to del from %s_%s", toString, pState->pTdbState->idstr, funcname); \ } \ } while (0); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 401a8b9e74..042ea373a1 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -157,7 +157,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) tEncodeStreamRetrieveReq(&encoder, &req); tEncoderClear(&encoder); - SRpcMsg rpcMsg = { .code = 0, .msgType = TDMT_STREAM_RETRIEVE, .pCont = buf, .contLen = sizeof(SMsgHead) + len }; + SRpcMsg rpcMsg = {.code = 0, .msgType = TDMT_STREAM_RETRIEVE, .pCont = buf, .contLen = sizeof(SMsgHead) + len}; if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) { ASSERT(0); goto CLEAR; From 6d04c4e2fd3ab3463c422f73cb298a51a9b586c2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:05:40 +0800 Subject: [PATCH 133/187] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6ec9020759..8cc791d1a6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1274,13 +1274,12 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); int32_t vgId = pTq->pStreamMeta->vgId; - tqDebug("vgId:%d recv dispatch rsp, code:%d", vgId, pMsg->code); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } else { - tqDebug("vgId:%d failed to find task:0x%x", vgId, taskId); + tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, taskId); return TSDB_CODE_INVALID_MSG; } } From 87deb36f8a3a6fe9c903e5be0f775478a0d15149 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:36:02 +0800 Subject: [PATCH 134/187] refactor: do some internal refactor. --- source/libs/executor/src/executor.c | 5 ++++- source/libs/executor/src/scanoperator.c | 10 ++++++---- source/libs/stream/src/streamExec.c | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fb35b211c9..ed8a4f3a3b 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -132,7 +132,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pOperator->status = OP_NOT_OPENED; SStreamScanInfo* pInfo = pOperator->info; - qDebug("s-task:%s set source blocks:%d", id, (int32_t)numOfBlocks); + + qDebug("s-task:%s in this batch, all %d blocks need to be processed and dump results", id, (int32_t)numOfBlocks); ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0); if (type == STREAM_INPUT__MERGED_SUBMIT) { @@ -140,6 +141,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); taosArrayPush(pInfo->pBlockLists, pReq); } + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { taosArrayPush(pInfo->pBlockLists, input); @@ -150,6 +152,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SPackedData tmp = { .pDataBlock = pDataBlock }; taosArrayPush(pInfo->pBlockLists, &tmp); } + pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } else { ASSERT(0); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2d2f377041..88f5642ef9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1797,9 +1797,10 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + const char* id = GET_TASKID(pTaskInfo); + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStreamScanInfo* pInfo = pOperator->info; qDebug("stream scan started, %s", GET_TASKID(pTaskInfo)); @@ -1922,7 +1923,9 @@ FETCH_NEXT_BLOCK: return NULL; } - int32_t current = pInfo->validBlockIndex++; + int32_t current = pInfo->validBlockIndex++; + qDebug("process %d/%d input data blocks, %s", current, (int32_t) total, id); + SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.parTbName[0]) { @@ -2057,7 +2060,6 @@ FETCH_NEXT_BLOCK: return pInfo->pUpdateRes; } - const char* id = GET_TASKID(pTaskInfo); SSDataBlock* pBlock = pInfo->pRes; SDataBlockInfo* pBlockInfo = &pBlock->info; int32_t totalBlocks = taosArrayGetSize(pInfo->pBlockLists); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index d6f5533db4..0970bbcf0c 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -16,8 +16,8 @@ #include "streamInc.h" // maximum allowed processed block batches. One block may include several submit blocks -#define MAX_STREAM_EXEC_BATCH_NUM 128 -#define MIN_STREAM_EXEC_BATCH_NUM 16 +#define MAX_STREAM_EXEC_BATCH_NUM 32 +#define MIN_STREAM_EXEC_BATCH_NUM 8 #define MAX_STREAM_RESULT_DUMP_THRESHOLD 1000 static int32_t updateCheckPointInfo (SStreamTask* pTask); From ccc86f991637e101a1544a5a409fca5049b39740 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:36:34 +0800 Subject: [PATCH 135/187] refactor: do some internal refactor. --- source/libs/stream/src/streamExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 0970bbcf0c..716b939e5f 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -18,7 +18,7 @@ // maximum allowed processed block batches. One block may include several submit blocks #define MAX_STREAM_EXEC_BATCH_NUM 32 #define MIN_STREAM_EXEC_BATCH_NUM 8 -#define MAX_STREAM_RESULT_DUMP_THRESHOLD 1000 +#define MAX_STREAM_RESULT_DUMP_THRESHOLD 100 static int32_t updateCheckPointInfo (SStreamTask* pTask); From 91de00597d64d89b41acb254eb09f30fdf67c874 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 22:12:14 +0800 Subject: [PATCH 136/187] refactor: do some internal refactor. --- source/libs/stream/src/streamExec.c | 36 ++++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 716b939e5f..c3dd848bc7 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -17,7 +17,7 @@ // maximum allowed processed block batches. One block may include several submit blocks #define MAX_STREAM_EXEC_BATCH_NUM 32 -#define MIN_STREAM_EXEC_BATCH_NUM 8 +#define MIN_STREAM_EXEC_BATCH_NUM 4 #define MAX_STREAM_RESULT_DUMP_THRESHOLD 100 static int32_t updateCheckPointInfo (SStreamTask* pTask); @@ -44,6 +44,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* if (numOfBlocks > 0) { SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(pItem, pTask, size, pRes); if (pStreamBlocks == NULL) { + qError("s-task:%s failed to create result stream data block, code:%s", pTask->id.idStr, tstrerror(terrno)); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return -1; } @@ -314,8 +315,13 @@ int32_t updateCheckPointInfo (SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } +/** + * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the + * appropriate batch of blocks should be handled in 5 to 10 sec. + */ int32_t streamExecForAll(SStreamTask* pTask) { - int32_t code = 0; + const char* id = pTask->id.idStr; + while (1) { int32_t batchSize = 1; int16_t times = 0; @@ -323,7 +329,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { SStreamQueueItem* pInput = NULL; // merge multiple input data if possible in the input queue. - qDebug("s-task:%s start to extract data block from inputQ", pTask->id.idStr); + qDebug("s-task:%s start to extract data block from inputQ", id); while (1) { if (streamTaskShouldPause(&pTask->status)) { @@ -338,7 +344,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { if (qItem == NULL) { if (pTask->taskLevel == TASK_LEVEL__SOURCE && batchSize < MIN_STREAM_EXEC_BATCH_NUM && times < 5) { times++; - taosMsleep(1); + taosMsleep(10); qDebug("===stream===try again batchSize:%d", batchSize); continue; } @@ -363,8 +369,10 @@ int32_t streamExecForAll(SStreamTask* pTask) { batchSize++; pInput = newRet; streamQueueProcessSuccess(pTask->inputQueue); + if (batchSize > MAX_STREAM_EXEC_BATCH_NUM) { - qDebug("maximum batch limit:%d reached, processing, %s", MAX_STREAM_EXEC_BATCH_NUM, pTask->id.idStr); + qDebug("s-task:%s maximum batch limit:%d reached, processing this batch of blocks", id, + MAX_STREAM_EXEC_BATCH_NUM); break; } } @@ -375,7 +383,6 @@ int32_t streamExecForAll(SStreamTask* pTask) { if (pInput) { streamFreeQitem(pInput); } - return 0; } @@ -385,7 +392,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { if (pTask->taskLevel == TASK_LEVEL__SINK) { ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); - qDebug("s-task:%s sink task start to sink %d blocks", pTask->id.idStr, batchSize); + qDebug("s-task:%s sink task start to sink %d blocks", id, batchSize); streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pInput); continue; } @@ -394,16 +401,16 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (pTask->taskLevel == TASK_LEVEL__SOURCE) { int8_t status = atomic_load_8(&pTask->status.taskStatus); if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__PAUSE) { - qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr, + qError("stream task wait for the end of fill history, s-task:%s, status:%d", id, atomic_load_8(&pTask->status.taskStatus)); - taosMsleep(2); + taosMsleep(100); } else { break; } } int64_t st = taosGetTimestampMs(); - qDebug("s-task:%s start to execute, block batches:%d", pTask->id.idStr, batchSize); + qDebug("s-task:%s start to process batch of blocks, num:%d", id, batchSize); { // set input @@ -417,21 +424,21 @@ int32_t streamExecForAll(SStreamTask* pTask) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); - qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, + qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)pInput; SArray* pBlockList = pBlock->blocks; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); + qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, id, numOfBlocks, pBlock->sourceVer); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput; SArray* pBlockList = pMerged->submits; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks); + qDebug("s-task:%s %p set submit input (merged), batch num:%d", id, pTask, numOfBlocks); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput; @@ -446,7 +453,8 @@ int32_t streamExecForAll(SStreamTask* pTask) { streamTaskExecImpl(pTask, pInput, &resSize, &totalBlocks); double el = (taosGetTimestampMs() - st) / 1000.0; - qDebug("s-task:%s exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", pTask->id.idStr, el, resSize / 1048576.0, totalBlocks); + qDebug("s-task:%s batch of input blocks exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", + id, el, resSize / 1048576.0, totalBlocks); streamFreeQitem(pInput); } From 6517ede04461b5f471c1859fbbd7d3963e0010df Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 1 Jun 2023 11:23:31 +0800 Subject: [PATCH 137/187] fix: fix interval operator ensure capacity logic --- source/libs/executor/src/executorInt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index e18906ad18..5872d019fa 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -827,7 +827,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - uint32_t newSize = pBlock->info.rows + pRow->numOfRows + (numOfRows - i) > 1 ? 1 : 0; + uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - i) > 1 ? 1 : 0); blockDataEnsureCapacity(pBlock, newSize); qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", newSize, pBlock->info.capacity, GET_TASKID(pTaskInfo)); From 706e35021733713389bdb065d064189e4fb89502 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jun 2023 14:36:42 +0800 Subject: [PATCH 138/187] fix(query): fix error in fill and partition. --- source/libs/executor/inc/tfill.h | 2 ++ source/libs/executor/src/filloperator.c | 28 +++++++++++++++++++------ source/libs/executor/src/tfill.c | 19 ++++++++++++++--- tests/script/tsim/query/partitionby.sim | 4 ++-- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 78b3cd2f40..79837480d7 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -120,6 +120,8 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts); +bool taosFillNotStarted(const SFillInfo* pFillInfo); SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, const struct SNodeListNode* val); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 839c845bac..3d1df6482e 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -139,6 +139,12 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { } SOperatorInfo* pDownstream = pOperator->pDownstream[0]; + + // the scan order may be different from the output result order for agg interval operator. + if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL) { + order = ((SIntervalAggOperatorInfo*) pDownstream->info)->resultTsOrder; + } + while (1) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { @@ -162,11 +168,24 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block pInfo->totalInputRows += pInfo->pRes->info.rows; - if (order == pInfo->pFillInfo->order) { + if (order == TSDB_ORDER_ASC) { + int64_t skey = pBlock->info.window.skey; + if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); + } + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); } else { + int64_t ekey = pBlock->info.window.ekey; + if (ekey > pInfo->pFillInfo->start) { + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); + } + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); } + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block pInfo->existNewGroupBlock = pBlock; @@ -256,11 +275,8 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); - STimeWindow w = {0}; - int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - - getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); - pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, + int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; + pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); if (order == TSDB_ORDER_ASC) { diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index e4d5d2c268..7cc50a70ab 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -517,11 +517,16 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) return; } + // the endKey is now the aligned time window value. truncate time window isn't correct. pFillInfo->end = endKey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval); - pFillInfo->end = taosTimeAdd(pFillInfo->end, pFillInfo->interval.interval, pFillInfo->interval.intervalUnit,pFillInfo->interval.precision); + +#if 0 + if (pFillInfo->order == TSDB_ORDER_ASC) { + ASSERT(pFillInfo->start <= pFillInfo->end); + } else { + ASSERT(pFillInfo->start >= pFillInfo->end); } +#endif pFillInfo->index = 0; pFillInfo->numOfRows = numOfRows; @@ -531,6 +536,13 @@ void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) pFillInfo->pSrcBlock = (SSDataBlock*)pInput; } +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts) { + pFillInfo->start = ts; + pFillInfo->currentKey = ts; +} + +bool taosFillNotStarted(const SFillInfo* pFillInfo) {return pFillInfo->start == pFillInfo->currentKey;} + bool taosFillHasMoreResults(SFillInfo* pFillInfo) { int32_t remain = taosNumOfRemainRows(pFillInfo); if (remain > 0) { @@ -565,6 +577,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } + numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); numOfRes += 1; diff --git a/tests/script/tsim/query/partitionby.sim b/tests/script/tsim/query/partitionby.sim index da7dfc0090..76d4f87908 100644 --- a/tests/script/tsim/query/partitionby.sim +++ b/tests/script/tsim/query/partitionby.sim @@ -121,7 +121,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 2 then +if $data00 != 4 then return -1 endi @@ -129,7 +129,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 8 then +if $data00 != 16 then return -1 endi From 21920721f3b7c34924d6c9971a8e8677618eb413 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 15:10:08 +0800 Subject: [PATCH 139/187] fix:make version compatible --- source/dnode/mnode/impl/src/mndTopic.c | 8 +++++--- source/dnode/vnode/src/tq/tqMeta.c | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 10bd7a7a8d..a2c0014245 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -28,7 +28,7 @@ #include "parser.h" #include "tname.h" -#define MND_TOPIC_VER_NUMBER 2 +#define MND_TOPIC_VER_NUMBER 3 #define MND_TOPIC_RESERVE_SIZE 64 SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); @@ -170,7 +170,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER; - if (sver != 1 && sver != 2) { + if (sver < 1 || sver > MND_TOPIC_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto TOPIC_DECODE_OVER; } @@ -197,7 +197,9 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pTopic->withMeta, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->stbUid, TOPIC_DECODE_OVER); - SDB_GET_BINARY(pRaw, dataPos, pTopic->stbName, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER); + if (sver >= 3) { + SDB_GET_BINARY(pRaw, dataPos, pTopic->stbName, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER); + } SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); if (pTopic->sql == NULL) { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 0edffd7f05..0021aa6ba7 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -65,7 +65,9 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tDecodeI64(pDecoder, &pHandle->execHandle.execTb.suid) < 0) return -1; - if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1; + if (!tDecodeIsEnd(pDecoder)){ + if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1; + } } tEndDecode(pDecoder); return 0; From 64edab63025818aa807737cab687f31d0c9206c9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jun 2023 07:51:52 +0000 Subject: [PATCH 140/187] refactor code --- source/libs/stream/src/streamBackendRocksdb.c | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 9d94a28c8a..eabec058d7 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1134,31 +1134,29 @@ int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key) { int32_t streamStateClear_rocksdb(SStreamState* pState) { qDebug("streamStateClear_rocksdb"); - SStateKey sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number}; - SStateKey eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number}; char sKeyStr[128] = {0}; char eKeyStr[128] = {0}; + SStateKey sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number}; + SStateKey eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number}; int sLen = stateKeyEncode(&sKey, sKeyStr); int eLen = stateKeyEncode(&eKey, eKeyStr); - char toStringStart[128] = {0}; - char toStringEnd[128] = {0}; - if (qDebugFlag & DEBUG_TRACE) { - stateKeyToString(&sKey, toStringStart); - stateKeyToString(&eKey, toStringEnd); - } - - char* err = NULL; if (pState->pTdbState->pHandle[1] != NULL) { + char* err = NULL; rocksdb_delete_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->writeOpts, pState->pTdbState->pHandle[1], sKeyStr, sLen, eKeyStr, eLen, &err); - } - // rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[0], sKeyStr, sLen, eKeyStr, - // eLen); - if (err != NULL) { - qWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); - taosMemoryFree(err); + if (err != NULL) { + char toStringStart[128] = {0}; + char toStringEnd[128] = {0}; + stateKeyToString(&sKey, toStringStart); + stateKeyToString(&eKey, toStringEnd); + + qWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); + taosMemoryFree(err); + } else { + rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[1], sKeyStr, sLen, eKeyStr, eLen); + } } return 0; From d20e0a06f6849e40bcfd74346729084009bfc3af Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 16:04:45 +0800 Subject: [PATCH 141/187] fix:make version compatible --- source/dnode/vnode/src/tq/tqMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 0021aa6ba7..25cfad4b36 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -341,7 +341,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - if(strcmp(handle.execHandle.execTb.qmsg, "") != 0) { + if(handle.execHandle.execTb.qmsg != NULL && strcmp(handle.execHandle.execTb.qmsg, "") != 0) { if (nodesStringToNode(handle.execHandle.execTb.qmsg, &handle.execHandle.execTb.node) != 0) { tqError("nodesStringToNode error in sub stable, since %s", terrstr()); return -1; From 69499b95826c9c973cb67a023a7ba8f67a6f17e4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 16:09:17 +0800 Subject: [PATCH 142/187] fix:make version compatible --- source/dnode/vnode/src/tq/tqMeta.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 25cfad4b36..239d65fdb7 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -37,7 +37,9 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tEncodeI64(pEncoder, pHandle->execHandle.execTb.suid) < 0) return -1; - if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1; + if (pHandle->execHandle.execTb.qmsg != NULL){ + if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1; + } } tEndEncode(pEncoder); return pEncoder->pos; From 64656a59d9a9f4fb45d3355cd927093900c75620 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Thu, 1 Jun 2023 16:28:32 +0800 Subject: [PATCH 143/187] reset sream fill block index --- source/libs/executor/src/filloperator.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 90b7f4a77e..519ddc1916 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -1002,9 +1002,10 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) { SSDataBlock* pBlock = pInfo->pSrcBlock; uint64_t groupId = pBlock->info.id.groupId; SSDataBlock* pRes = pInfo->pRes; + SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); + TSKEY* tsCol = (TSKEY*)pTsCol->pData; pRes->info.id.groupId = groupId; - SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); - TSKEY* tsCol = (TSKEY*)pTsCol->pData; + pInfo->srcRowIndex++; if (pInfo->srcRowIndex == 0) { keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); @@ -1242,7 +1243,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { SSDataBlock* fillResult = NULL; SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { - if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows) { + if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) { // If there are delete datablocks, we receive them first. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -1281,7 +1282,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { case STREAM_PULL_DATA: { doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - pInfo->srcRowIndex = 0; + pInfo->srcRowIndex = -1; } break; case STREAM_CREATE_CHILD_TABLE: { return pBlock; @@ -1497,7 +1498,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi goto _error; } - pInfo->srcRowIndex = 0; + pInfo->srcRowIndex = -1; setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = From bb772e2062c5b17e73a7944dbe67600868aedb2e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 17:03:02 +0800 Subject: [PATCH 144/187] fix:make version compatible --- source/dnode/mnode/impl/src/mndTopic.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index a2c0014245..f1ee7bca3b 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -924,13 +924,12 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ SStbObj *pStb = mndAcquireStb(pMnode, pTopic->stbName); if (pStb == NULL) { - terrno = TSDB_CODE_MND_STB_NOT_EXIST; - taosMemoryFree(schemaJson); - return -1; + STR_TO_VARSTR(schemaJson, "NULL"); + mError("mndRetrieveTopic mndAcquireStb null stbName:%s", pTopic->stbName); + }else{ + schemaToJson(pStb->pColumns, pStb->numOfColumns, schemaJson); + mndReleaseStb(pMnode, pStb); } - schemaToJson(pStb->pColumns, pStb->numOfColumns, schemaJson); - - mndReleaseStb(pMnode, pStb); }else{ STR_TO_VARSTR(schemaJson, "NULL"); } From 2935ebcfc4d48d3ade0737f87fa7381872a34b06 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Thu, 1 Jun 2023 17:26:03 +0800 Subject: [PATCH 145/187] set session update info --- source/libs/executor/src/timewindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2cdf9bb15c..2eb8d297ac 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2904,7 +2904,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; - if ((!pScanInfo->igCheckUpdate || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) && !pScanInfo->pUpdateInfo) { + if (!pScanInfo->pUpdateInfo) { pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); } pScanInfo->twAggSup = *pTwSup; From 1036259a7b707a48902b02cb56a563c432ca37d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jun 2023 18:25:05 +0800 Subject: [PATCH 146/187] fix(query): fix error in fill. --- source/common/src/ttime.c | 1 + source/libs/executor/src/filloperator.c | 87 ++++++++++++++++++------- tests/script/tsim/parser/function.sim | 3 + 3 files changed, 69 insertions(+), 22 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 0aeb420b67..d8c43747f7 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -738,6 +738,7 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char ekey = skey; skey = tmp; } + if (unit != 'n' && unit != 'y') { return (int32_t)((ekey - skey) / interval); } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 3d1df6482e..e257879a04 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -61,6 +61,7 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); @@ -72,14 +73,16 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; getTableScanInfo(pOperator, &order, &scanFlag, false); - - int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); blockDataCleanup(pInfo->pRes); doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey); + revisedFillStartKey(pInfo, pInfo->existNewGroupBlock); + + int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows; @@ -119,6 +122,55 @@ void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int pInfo->pRes->info.id.groupId = pBlock->info.id.groupId; } +// todo refactor: decide the start key according to the query time range. +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { + int32_t order = pInfo->pFillInfo->order; + + if (order == TSDB_ORDER_ASC) { + int64_t skey = pBlock->info.window.skey; + if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); + } else if (pInfo->pFillInfo->start < skey) { + int64_t t = skey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev < pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } else { + int64_t ekey = pBlock->info.window.ekey; + if (ekey > pInfo->pFillInfo->start) { + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); + } else if (ekey < pInfo->pFillInfo->start) { + int64_t t = ekey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev < pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } +} + static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -164,28 +216,16 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); - if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) { + if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { + if (pInfo->curGroupId == 0) { + revisedFillStartKey(pInfo, pBlock); + } + pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block pInfo->totalInputRows += pInfo->pRes->info.rows; - if (order == TSDB_ORDER_ASC) { - int64_t skey = pBlock->info.window.skey; - if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the - ASSERT( taosFillNotStarted(pInfo->pFillInfo)); - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); - } - - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); - } else { - int64_t ekey = pBlock->info.window.ekey; - if (ekey > pInfo->pFillInfo->start) { - ASSERT( taosFillNotStarted(pInfo->pFillInfo)); - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); - } - - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); - } - + int64_t ts = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block pInfo->existNewGroupBlock = pBlock; @@ -276,6 +316,9 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; + +// STimeWindow w = {0}; +// getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 7f69aa2d02..120c4b8148 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -954,11 +954,14 @@ endi print =========================>TD-5190 sql select _wstart, stddev(f1) from st1 where ts>'2021-07-01 1:1:1' and ts<'2021-07-30 00:00:00' interval(1d) fill(NULL); if $rows != 29 then + print expect 29, actual: $rows return -1 endi + if $data00 != @21-07-01 00:00:00.000@ then return -1 endi + if $data01 != NULL then return -1 endi From 56782a5d41eca1b85bf66ec1c48362188c351c43 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 18:29:16 +0800 Subject: [PATCH 147/187] fix:open task case & modify mqRebVgReq encode/decode style --- include/common/tmsg.h | 60 +++++++++++++--------- source/dnode/mnode/impl/src/mndSubscribe.c | 19 +++++-- source/dnode/vnode/src/tq/tq.c | 19 ++++--- tests/parallel_test/cases.task | 2 +- 4 files changed, 65 insertions(+), 35 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b4674aba54..d78e771fcf 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2807,39 +2807,49 @@ typedef struct { int64_t suid; } SMqRebVgReq; -static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->leftForVer); - tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); - tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); - tlen += taosEncodeString(buf, pReq->subKey); - tlen += taosEncodeFixedI8(buf, pReq->subType); - tlen += taosEncodeFixedI8(buf, pReq->withMeta); +static FORCE_INLINE int tEncodeSMqRebVgReq(SEncoder *pCoder, const SMqRebVgReq* pReq) { + if (tStartEncode(pCoder) < 0) return -1; + if (tEncodeI64(pCoder, pReq->leftForVer) < 0) return -1; + if (tEncodeI32(pCoder, pReq->vgId) < 0) return -1; + if (tEncodeI64(pCoder, pReq->oldConsumerId) < 0) return -1; + if (tEncodeI64(pCoder, pReq->newConsumerId) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->subKey) < 0) return -1; + if (tEncodeI8(pCoder, pReq->subType) < 0) return -1; + if (tEncodeI8(pCoder, pReq->withMeta) < 0) return -1; + if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { - tlen += taosEncodeString(buf, pReq->qmsg); + if (tEncodeCStr(pCoder, pReq->qmsg) < 0) return -1; } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { - tlen += taosEncodeFixedI64(buf, pReq->suid); - tlen += taosEncodeString(buf, pReq->qmsg); + if (tEncodeI64(pCoder, pReq->suid) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->qmsg) < 0) return -1; } - return tlen; + tEndEncode(pCoder); + return 0; } -static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->leftForVer); - buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); - buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); - buf = taosDecodeStringTo(buf, pReq->subKey); - buf = taosDecodeFixedI8(buf, &pReq->subType); - buf = taosDecodeFixedI8(buf, &pReq->withMeta); +static FORCE_INLINE int tDecodeSMqRebVgReq(SDecoder *pCoder, SMqRebVgReq* pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->leftForVer) < 0) return -1; + + if (tDecodeI32(pCoder, &pReq->vgId) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->oldConsumerId) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->newConsumerId) < 0) return -1; + if (tDecodeCStrTo(pCoder, pReq->subKey) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->subType) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->withMeta) < 0) return -1; + if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { - buf = taosDecodeString(buf, &pReq->qmsg); + if (tDecodeCStr(pCoder, &pReq->qmsg) < 0) return -1; } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { - buf = taosDecodeFixedI64(buf, &pReq->suid); - buf = taosDecodeString(buf, &pReq->qmsg); + if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; + if (!tDecodeIsEnd(pCoder)){ + if (tDecodeCStr(pCoder, &pReq->qmsg) < 0) return -1; + } } - return (void*)buf; + + tEndDecode(pCoder); + return 0; } typedef struct { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e62102fa77..74421afa33 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -111,7 +111,14 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri req.suid = pSub->stbUid; tstrncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); - int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); + int32_t tlen = 0; + int32_t ret = 0; + tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret); + if (ret < 0) { + return -1; + } + + tlen += sizeof(SMsgHead); void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -123,8 +130,14 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri pMsgHead->contLen = htonl(tlen); pMsgHead->vgId = htonl(pRebVg->pVgEp->vgId); - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqRebVgReq(&abuf, &req); + SEncoder encoder = {0}; + tEncoderInit(&encoder, POINTER_SHIFT(buf, sizeof(SMsgHead)), tlen); + if (tEncodeSMqRebVgReq(&encoder, &req) < 0) { + taosMemoryFreeClear(buf); + tEncoderClear(&encoder); + return -1; + } + tEncoderClear(&encoder); *pBuf = buf; *pLen = tlen; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 11760b5fd0..35055a00e3 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -633,7 +633,16 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { int ret = 0; SMqRebVgReq req = {0}; - tDecodeSMqRebVgReq(msg, &req); + SDecoder dc = {0}; + + tDecoderInit(&dc, msg, msgLen); + + // decode req + if (tDecodeSMqRebVgReq(&dc, &req) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&dc); + return -1; + } SVnode* pVnode = pTq->pVnode; int32_t vgId = TD_VID(pVnode); @@ -680,8 +689,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - pHandle->execHandle.execCol.qmsg = req.qmsg; - req.qmsg = NULL; + pHandle->execHandle.execCol.qmsg = taosStrdup(req.qmsg);; pHandle->execHandle.task = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, &pHandle->execHandle.numOfCols, req.newConsumerId); @@ -701,8 +709,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); pHandle->execHandle.execTb.suid = req.suid; - pHandle->execHandle.execTb.qmsg = req.qmsg; - req.qmsg = NULL; + pHandle->execHandle.execTb.qmsg = taosStrdup(req.qmsg); if(strcmp(pHandle->execHandle.execTb.qmsg, "") != 0) { if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { @@ -766,7 +773,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg end: taosWUnLockLatch(&pTq->lock); - taosMemoryFree(req.qmsg); + tDecoderClear(&dc); return ret; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6a9a2b609a..0b235d0dc8 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -564,7 +564,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel.py -#,,n,system-test,python3 ./test.py -f 0-others/compatibility.py +,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py From 0315e895918de6cad123a1e136c45dbbbbe38402 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jun 2023 11:13:01 +0000 Subject: [PATCH 148/187] Avoid creating the same ID task multiple times --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- source/libs/stream/src/streamMeta.c | 48 ++++++++++--------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index eabec058d7..bd72566e5a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -719,7 +719,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t qDebug("succ to open rocksdb cf"); } // close default cf - rocksdb_column_family_handle_destroy(cfHandle[0]); + if (((rocksdb_column_family_handle_t**)cfHandle)[0] != 0) rocksdb_column_family_handle_destroy(cfHandle[0]); rocksdb_options_destroy(cfOpts[0]); handle->db = db; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 98e63f7f51..e3f8b12367 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -205,24 +205,25 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { // add to the ready tasks hash map, not the restored tasks hash map int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { - if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { - tFreeStreamTask(pTask); - return -1; - } - - if (streamMetaSaveTask(pMeta, pTask) < 0) { - tFreeStreamTask(pTask); - return -1; - } - - if (streamMetaCommit(pMeta) < 0) { - tFreeStreamTask(pTask); - return -1; - } - void* p = taosHashGet(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); if (p == NULL) { + if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { + tFreeStreamTask(pTask); + return -1; + } + + if (streamMetaSaveTask(pMeta, pTask) < 0) { + tFreeStreamTask(pTask); + return -1; + } + + if (streamMetaCommit(pMeta) < 0) { + tFreeStreamTask(pTask); + return -1; + } taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); + } else { + return 0; } taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES); @@ -359,18 +360,19 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { tDecodeStreamTask(&decoder, pTask); tDecoderClear(&decoder); - if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCur); - return -1; - } - + // remove duplicate void* p = taosHashGet(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); if (p == NULL) { + if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + return -1; + } taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); + } else { + continue; } - if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, sizeof(void*)) < 0) { tdbFree(pKey); tdbFree(pVal); From 32c3cfd51a767baea59c756e4302c5410f36ad03 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jun 2023 11:13:27 +0000 Subject: [PATCH 149/187] Avoid creating the same ID task multiple times --- include/libs/stream/tstream.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 1d4bbf073e..8316e6ef50 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -206,7 +206,7 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { void* streamQueueNextItem(SStreamQueue* queue); SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type); -void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); +void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); SStreamDataSubmit* streamSubmitBlockClone(SStreamDataSubmit* pSubmit); @@ -284,7 +284,7 @@ struct SStreamTask { int16_t dispatchMsgType; SStreamStatus status; int32_t selfChildId; - int32_t nodeId; // vgroup id + int32_t nodeId; // vgroup id SEpSet epSet; SCheckpointInfo chkInfo; STaskExec exec; @@ -346,12 +346,14 @@ typedef struct SStreamMeta { void* streamBackend; int32_t streamBackendId; int64_t streamBackendRid; + SHashObj* pTaskBackendUnique; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); -SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, SArray* pTaskList); +SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, + SArray* pTaskList); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); void tFreeStreamTask(SStreamTask* pTask); From 1571ea844af5bdc52407326b37d081b1c9d55d24 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jun 2023 11:25:55 +0000 Subject: [PATCH 150/187] Avoid creating the same ID task multiple times --- source/libs/stream/src/streamMeta.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index e3f8b12367..8c26052fdb 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -367,16 +367,22 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); + taosMemoryFree(pTask); return -1; } taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); } else { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + taosMemoryFree(pTask); continue; } if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, sizeof(void*)) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); + taosMemoryFree(pTask); return -1; } From 403617976c500b77066519e752e6dffbff7b8e48 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jun 2023 22:45:03 +0800 Subject: [PATCH 151/187] fix(query): fix error in fill. --- source/libs/executor/src/filloperator.c | 40 ++++++++++++------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index e257879a04..75bfe7b671 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -61,24 +61,24 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; -static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock); +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; SSDataBlock* pResBlock = pInfo->pFinalRes; - int32_t order = TSDB_ORDER_ASC; +// int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag, false); +// getTableScanInfo(pOperator, &order, &scanFlag, false); taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); blockDataCleanup(pInfo->pRes); doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); - revisedFillStartKey(pInfo, pInfo->existNewGroupBlock); + revisedFillStartKey(pInfo, pInfo->existNewGroupBlock, order); int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey; taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); @@ -93,7 +93,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp } static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows; taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows); @@ -103,7 +103,7 @@ static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOpera // handle the cached new group data block if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); } } @@ -123,9 +123,7 @@ void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int } // todo refactor: decide the start key according to the query time range. -static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { - int32_t order = pInfo->pFillInfo->order; - +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order) { if (order == TSDB_ORDER_ASC) { int64_t skey = pBlock->info.window.skey; if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the @@ -137,7 +135,7 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { while(1) { int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - if (prev < pInfo->pFillInfo->start) { + if (prev <= pInfo->pFillInfo->start) { t = prev; break; } @@ -158,7 +156,7 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { while(1) { int64_t prev = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - if (prev < pInfo->pFillInfo->start) { + if (prev >= pInfo->pFillInfo->start) { t = prev; break; } @@ -184,12 +182,6 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { int32_t scanFlag = MAIN_SCAN; getTableScanInfo(pOperator, &order, &scanFlag, false); - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); - if (pResBlock->info.rows > 0) { - pResBlock->info.id.groupId = pInfo->curGroupId; - return pResBlock; - } - SOperatorInfo* pDownstream = pOperator->pDownstream[0]; // the scan order may be different from the output result order for agg interval operator. @@ -197,6 +189,12 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { order = ((SIntervalAggOperatorInfo*) pDownstream->info)->resultTsOrder; } + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); + if (pResBlock->info.rows > 0) { + pResBlock->info.id.groupId = pInfo->curGroupId; + return pResBlock; + } + while (1) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { @@ -218,7 +216,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { if (pInfo->curGroupId == 0) { - revisedFillStartKey(pInfo, pBlock); + revisedFillStartKey(pInfo, pBlock, order); } pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block @@ -249,7 +247,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { return pResBlock; } - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; @@ -257,7 +255,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { } else if (pInfo->existNewGroupBlock) { // try next group blockDataCleanup(pResBlock); - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows > pResultInfo->threshold) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; From bbf3f1ccd92af410c0f50bcb6487693cb862ea38 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 00:22:35 +0800 Subject: [PATCH 152/187] fix(query): add more check. --- source/libs/executor/src/filloperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 75bfe7b671..e2862b19c7 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -215,7 +215,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { - if (pInfo->curGroupId == 0) { + if (pInfo->curGroupId == 0 && taosFillNotStarted(pInfo->pFillInfo)) { revisedFillStartKey(pInfo, pBlock, order); } From 420ae73728044b13fa51743adebfba303240e480 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 00:35:45 +0000 Subject: [PATCH 153/187] Avoid creating the same ID task multiple times --- source/libs/stream/src/streamBackendRocksdb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index bd72566e5a..c743ecf7f7 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -128,7 +128,9 @@ void* streamBackendInit(const char* path) { */ streamStateOpenBackendCf(pHandle, (char*)path, cfs, nCf); } - rocksdb_list_column_families_destroy(cfs, nCf); + if (cfs != NULL) { + rocksdb_list_column_families_destroy(cfs, nCf); + } return (void*)pHandle; _EXIT: From 274771d7b4cb5329ba1267633e99d527b5a42846 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 2 Jun 2023 08:59:00 +0800 Subject: [PATCH 154/187] TD-24565 --- source/libs/executor/src/scanoperator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 28abee4342..39907be6d2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1072,6 +1072,7 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t ver) { pTableScanInfo->base.cond.twindows = *pWin; + pTableScanInfo->base.cond.startVersion = 0; pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; @@ -1194,6 +1195,7 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ } STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; + qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, pInfo->pUpdateInfo->maxDataVersion); resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; return true; @@ -1889,6 +1891,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } else { + pInfo->pUpdateInfo->maxDataVersion = pTaskInfo->streamInfo.fillHistoryVer2; doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); } } From becc5c51a40c1d1e9f14b65f54eef9e3f939f33a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 02:35:15 +0000 Subject: [PATCH 155/187] fix compatibility --- source/libs/stream/src/streamBackendRocksdb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c8a6597bad..c54f055e6a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1924,17 +1924,17 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; - STREAM_STATE_PUT_ROCKSDB(pState, "default", &key, pVal, pVLen); + STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) { int code = 0; - STREAM_STATE_GET_ROCKSDB(pState, "default", &key, pVal, pVLen); + STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } int32_t streamDefaultDel_rocksdb(SStreamState* pState, const void* key) { int code = 0; - STREAM_STATE_DEL_ROCKSDB(pState, "default", &key); + STREAM_STATE_DEL_ROCKSDB(pState, "default", key); return code; } From 27f791ac7fc071bc7899d0a0188137686529eaf0 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 2 Jun 2023 11:04:02 +0800 Subject: [PATCH 156/187] reset ud table name --- source/dnode/vnode/src/tq/tqSink.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6e02d5e21b..db1b5ed902 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -294,6 +294,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d char* ctbName = pDataBlock->info.parTbName; if (!ctbName[0]) { + memset(ctbName, 0, TSDB_TABLE_NAME_LEN); if (res == TSDB_CODE_SUCCESS) { memcpy(ctbName, pTableSinkInfo->tbName, strlen(pTableSinkInfo->tbName)); } else { From 5c64a730a9668f2768a13c9255f4a73e6b087046 Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 11:17:38 +0800 Subject: [PATCH 157/187] enchance: change goes as static link --- cmake/cmake.options | 2 +- cmake/cmake.platform | 10 ---------- packaging/tools/makepkg.sh | 3 +++ 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index b00ae14715..555b72cbdf 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -80,7 +80,7 @@ ENDIF () option( BUILD_GEOS "If build geos on Windows" - OFF + ON ) option( diff --git a/cmake/cmake.platform b/cmake/cmake.platform index f9faf7316c..ba747c6134 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -56,17 +56,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(TD_DARWIN TRUE) SET(OSTYPE "macOS") - execute_process(COMMAND geos-config --cflags OUTPUT_VARIABLE GEOS_CFLAGS) - execute_process(COMMAND geos-config --ldflags OUTPUT_VARIABLE GEOS_LDFLAGS) - string(SUBSTRING ${GEOS_CFLAGS} 2 -1 GEOS_CFLAGS) - string(REGEX REPLACE "\n" "" GEOS_CFLAGS ${GEOS_CFLAGS}) - string(SUBSTRING ${GEOS_LDFLAGS} 2 -1 GEOS_LDFLAGS) - string(REGEX REPLACE "\n" "" GEOS_LDFLAGS ${GEOS_LDFLAGS}) - MESSAGE("GEOS_CFLAGS "${GEOS_CFLAGS}) - MESSAGE("GEOS_LDFLAGS "${GEOS_LDFLAGS}) ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") - INCLUDE_DIRECTORIES(${GEOS_CFLAGS}) - LINK_DIRECTORIES(${GEOS_LDFLAGS}) IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") MESSAGE("Current system arch is arm64") diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 2f1e803689..7cfd9aa664 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -112,9 +112,11 @@ fi if [ "$osType" == "Darwin" ]; then lib_files="${build_dir}/lib/libtaos.${version}.dylib" wslib_files="${build_dir}/lib/libtaosws.dylib" + rocksdb_lib_files="${build_dir}/lib/librocksdb.dylib.8.1.1" else lib_files="${build_dir}/lib/libtaos.so.${version}" wslib_files="${build_dir}/lib/libtaosws.so" + rocksdb_lib_files="${build_dir}/lib/librocksdb.so.8.1.1" fi header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" @@ -337,6 +339,7 @@ fi # Copy driver mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : +[ -f ${rocksdb_lib_files} ] && cp ${rocksdb_lib_files} ${install_dir}/driver || : # Copy connector if [ "$verMode" == "cluster" ]; then From 27b7d1ec8818672d93e6e0d5c47c084b0b140ae1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 11:17:44 +0800 Subject: [PATCH 158/187] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/executor/src/scanoperator.c | 4 ++-- source/libs/stream/src/stream.c | 10 +++++----- source/libs/stream/src/streamDispatch.c | 6 +++--- source/libs/stream/src/streamExec.c | 12 ++++++------ 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8cc791d1a6..5e7e21e7bd 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1086,7 +1086,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - qDebug("s-task:%s set the start wal offset to be:%"PRId64, pTask->id.idStr, sversion); + qDebug("s-task:%s set start wal scan start ver:%"PRId64, pTask->id.idStr, sversion); walReaderSeekVer(pTask->exec.pWalReader, sversion); pTask->chkInfo.currentVer = sversion; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 88f5642ef9..80a7505c77 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2005,7 +2005,7 @@ FETCH_NEXT_BLOCK: // printDataBlock(pBlock, "stream scan recv"); return pBlock; } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) { - qDebug("scan mode %d", pInfo->scanMode); + qDebug("stream scan mode:%d, %s", pInfo->scanMode, id); switch (pInfo->scanMode) { case STREAM_SCAN_FROM_RES: { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; @@ -2122,7 +2122,7 @@ FETCH_NEXT_BLOCK: pInfo->numOfExec++; pOperator->resultInfo.totalRows += pBlockInfo->rows; - qDebug("stream scan get source rows:%" PRId64", %s", pBlockInfo->rows, id); + qDebug("stream scan completed, and return source rows:%" PRId64", %s", pBlockInfo->rows, id); if (pBlockInfo->rows > 0) { return pBlock; } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index acc69c5a2b..7457b2197e 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -296,11 +296,8 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { if (type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* px = (SStreamDataSubmit*)pItem; - qDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr, - px->submit.msgLen, px->submit.ver, total, size); - if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && tInputQueueIsFull(pTask)) { - qError("s-task:%s input queue is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) abort", + qError("s-task:%s input queue is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); streamDataSubmitDestroy(px); @@ -314,9 +311,12 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { taosFreeQitem(pItem); return code; } + + qDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr, + px->submit.msgLen, px->submit.ver, total, size + px->submit.msgLen/1048576.0); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { - if (/*(pTask->taskLevel == TASK_LEVEL__SOURCE) && */(tInputQueueIsFull(pTask))) { + if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && (tInputQueueIsFull(pTask))) { qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 1e939cb071..bd6a013de2 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -283,7 +283,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov msg.info.noResp = 1; tmsgSendReq(pEpSet, &msg); - qDebug("s-task:%s dispatch recover finish msg to taskId:%d node %d: recover finish msg", pTask->id.idStr, + qDebug("s-task:%s dispatch recover finish msg to downstream taskId:0x%x node %d: recover finish msg", pTask->id.idStr, pReq->taskId, vgId); return 0; @@ -414,7 +414,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat req.taskId = downstreamTaskId; - qDebug("s-task:%s (child taskId:%d) fix-dispatch blocks:%d to down stream s-task:%d in vgId:%d", pTask->id.idStr, + qDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to down stream s-task:0x%x in vgId:%d", pTask->id.idStr, pTask->selfChildId, numOfBlocks, downstreamTaskId, vgId); code = doSendDispatchMsg(pTask, &req, vgId, pEpSet); @@ -514,7 +514,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { return 0; } - qDebug("s-task:%s start to dispatch msg, output status:%d", pTask->id.idStr, pTask->outputStatus); + qDebug("s-task:%s start to dispatch msg, set output status:%d", pTask->id.idStr, pTask->outputStatus); SStreamDataBlock* pDispatchedBlock = streamQueueNextItem(pTask->outputQueue); if (pDispatchedBlock == NULL) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index c3dd848bc7..813bc50b87 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -236,11 +236,11 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { taosFreeQitem(qRes); return code; } - - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - qDebug("s-task:%s scan exec dispatch blocks:%d", pTask->id.idStr, batchCnt); - streamDispatchStreamBlock(pTask); - } +// +// if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { +// qDebug("s-task:%s scan exec dispatch blocks:%d", pTask->id.idStr, batchCnt); +// streamDispatchStreamBlock(pTask); +// } if (finished) { break; @@ -438,7 +438,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { SArray* pBlockList = pMerged->submits; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s %p set submit input (merged), batch num:%d", id, pTask, numOfBlocks); + qDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d", id, pTask, numOfBlocks); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput; From 63d9a5dcf37220262e11db2c3a96c2626fa798d7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 04:08:08 +0000 Subject: [PATCH 159/187] avoid mem leak --- source/libs/stream/src/tstreamFileState.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index f531f65565..54832b042a 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -16,12 +16,12 @@ #include "tstreamFileState.h" #include "query.h" +#include "storageapi.h" #include "streamBackendRocksdb.h" #include "taos.h" #include "tcommon.h" #include "thash.h" #include "tsimplehash.h" -#include "storageapi.h" #define FLUSH_RATIO 0.5 #define FLUSH_NUM 4 @@ -420,6 +420,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { return TSDB_CODE_FAILED; } sscanf(val, "%" PRId64 "", &maxCheckPointId); + taosMemoryFree(val); } for (int64_t i = maxCheckPointId; i > 0; i--) { char buf[128] = {0}; @@ -435,9 +436,11 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (ts < mark) { // statekey winkey.ts < mark forceRemoveCheckpoint(pFileState, i); + taosMemoryFreeClear(val); break; } else { } + taosMemoryFree(val); } return code; } From f1bd829977b43a0bbb12b4f9acdfb13d57f4a87d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 13:10:54 +0800 Subject: [PATCH 160/187] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tq/tq.c | 3 ++- source/dnode/vnode/src/tq/tqMeta.c | 3 ++- source/dnode/vnode/src/tq/tqRead.c | 19 +++++++++++-------- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/scanoperator.c | 5 +++-- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamExec.c | 4 ++-- 8 files changed, 23 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index fbe2f4809c..8003419ec9 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -200,7 +200,7 @@ STqReader *tqReaderOpen(SVnode *pVnode); void tqReaderClose(STqReader *); void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); -int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList, const char* id); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5e7e21e7bd..a6bed26884 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -708,8 +708,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } + pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); - tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); + tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList, NULL); taosArrayDestroy(tbUidList); buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index be5b5706e2..a720e556e6 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -344,8 +344,9 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } + handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); - tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); + tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList, NULL); taosArrayDestroy(tbUidList); buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index dbb7d564f8..f15a9898b5 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -394,8 +394,8 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) { SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pReader->tbIdHash == NULL) { - SSDataBlock* pRes = NULL; - int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); + SSDataBlock* pRes = NULL; + int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { return true; } @@ -457,7 +457,7 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { - tqDebug("tq reader next data block, len:%d ver:%" PRId64 " index:%d/%d, %s", pReader->msg.msgLen, pReader->msg.ver, + tqDebug("try next data block, len:%d ver:%" PRId64 " index:%d/%d, %s", pReader->msg.msgLen, pReader->msg.ver, pReader->nextBlk, numOfBlocks, idstr); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); @@ -467,10 +467,11 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); if (ret != NULL) { - tqDebug("tq reader block found, ver:%" PRId64 ", uid:%" PRId64, pReader->msg.ver, pSubmitTbData->uid); + tqDebug("block found, ver:%" PRId64 ", uid:%" PRId64", %s", pReader->msg.ver, pSubmitTbData->uid, idstr); return true; } else { - tqDebug("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid); + tqDebug("discard submit block, uid:%" PRId64 ", total queried tables:%d continue %s", pSubmitTbData->uid, + taosHashGetSize(pReader->tbIdHash), idstr); } pReader->nextBlk++; @@ -604,7 +605,6 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) { tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); - SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); SSDataBlock* pBlock = pReader->pResBlock; @@ -612,6 +612,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* blockDataCleanup(pBlock); + int32_t vgId = pReader->pWalReader->pWal->cfg.vgId; int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; @@ -628,7 +629,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64 "version %d, possibly dropped table", - pReader->pWalReader->pWal->cfg.vgId, suid, uid, pReader->cachedSchemaVer); + vgId, suid, uid, pReader->cachedSchemaVer); pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -642,6 +643,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* if (blockDataGetNumOfCols(pBlock) == 0) { int32_t code = buildResSDataBlock(pReader->pResBlock, pReader->pSchemaWrapper, pReader->pColIdList); if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d failed to build data block, code:%s", vgId, tstrerror(code)); return code; } } @@ -998,7 +1000,7 @@ FAIL: void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } -int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { +int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList, const char* id) { if (pReader->tbIdHash) { taosHashClear(pReader->tbIdHash); } else { @@ -1015,6 +1017,7 @@ int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } + tqDebug("s-task:%s %d tables are set to be queried target table", id, (int32_t) taosArrayGetSize(tbUidList)); return 0; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ed8a4f3a3b..4d02362904 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -401,7 +401,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI int32_t code = 0; if (isAdd) { - qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id); + qDebug("try to add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id); } // traverse to the stream scanner node to add this table id diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 80a7505c77..5f09b0c67a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2308,7 +2308,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SArray* pColIds = NULL; SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + const char* idstr = pTaskInfo->id.str; if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2419,7 +2420,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys // set the extract column id to streamHandle pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds); SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); - code = pAPI->tqReaderFn.tqReaderSetQueryTableList(pInfo->tqReader, tableIdList); + code = pAPI->tqReaderFn.tqReaderSetQueryTableList(pInfo->tqReader, tableIdList, idstr); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index bd6a013de2..9501718416 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -318,7 +318,7 @@ int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, in msg.pCont = buf; msg.msgType = pTask->dispatchMsgType; - qDebug("dispatch from s-task:%s to taskId:0x%x vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); + qDebug("s-task:%s dispatch msg to taskId:0x%x vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); tmsgSendReq(pEpSet, &msg); code = 0; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 813bc50b87..c28cd6f334 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -133,7 +133,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i taosArrayPush(pRes, &block); - qDebug("s-task:%s (child %d) executed and get block, total blocks:%d, size:%.2fMiB", pTask->id.idStr, + qDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, pTask->selfChildId, numOfBlocks, size / 1048576.0); // current output should be dispatched to down stream nodes @@ -371,7 +371,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { streamQueueProcessSuccess(pTask->inputQueue); if (batchSize > MAX_STREAM_EXEC_BATCH_NUM) { - qDebug("s-task:%s maximum batch limit:%d reached, processing this batch of blocks", id, + qDebug("s-task:%s batch size limit:%d reached, start to process blocks", id, MAX_STREAM_EXEC_BATCH_NUM); break; } From dca2abd8d1fccd7688f0df6bec66db368fe59ffb Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Jun 2023 13:57:54 +0800 Subject: [PATCH 161/187] feat: change get taos-tools with 3.0 branch --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d9d2f12069..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 4378702 + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 3b69027325f59d03ed7893fdbb326e3392244c13 Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 14:01:25 +0800 Subject: [PATCH 162/187] enhance: supply librocksdb.so install and uninstall --- packaging/tools/install.sh | 9 +++++++++ packaging/tools/post.sh | 6 ++++++ packaging/tools/remove.sh | 3 +++ 3 files changed, 18 insertions(+) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 1b47b10520..9452e48d33 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -250,18 +250,27 @@ function install_lib() { # Remove links ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : + ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : #${csudo}rm -rf ${v15_java_app_dir} || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib_link_dir}/librocksdb.so.8 + ${csudo}ln -sf ${lib_link_dir}/librocksdb.so.8 ${lib_link_dir}/librocksdb.so + + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib64_link_dir}/librocksdb.so.8 || : + ${csudo}ln -sf ${lib64_link_dir}/librocksdb.so.8 ${lib64_link_dir}/librocksdb.so || : + [ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index fc392c9684..4a8dda1d30 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -202,10 +202,15 @@ function install_lib() { log_print "start install lib from ${lib_dir} to ${lib_link_dir}" ${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || : + + #rocksdb + [ -f ${lib_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb* || : + [ -f ${lib64_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib64_link_dir}/librocksdb* || : [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : + ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 @@ -214,6 +219,7 @@ function install_lib() { if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.${lib_file_ext} ]]; then ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 + ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib64_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} 2>>${install_log_path} fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 6c671473bf..a17b29983c 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -142,11 +142,14 @@ function clean_local_bin() { function clean_lib() { # Remove link ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : + } function clean_header() { From eb344891da96d8d58746707ca06eae7a3f479188 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Jun 2023 14:03:53 +0800 Subject: [PATCH 163/187] feat: get taos-tools code from main branch --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d9d2f12069..9bbda8309f 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 4378702 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From f59161464d7f596418b34915d3bd51368675e50c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 06:48:26 +0000 Subject: [PATCH 164/187] avoid mem leak --- source/libs/executor/src/scanoperator.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 146552b058..6d17672513 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +// clang-format off + #include "executorInt.h" #include "filter.h" #include "function.h" @@ -2444,6 +2446,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys int32_t len = 0; pAPI->stateStore.streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len); streamScanOperatorDecode(buff, len, pInfo); + taosMemoryFree(buff); } setOperatorInfo(pOperator, STREAM_SCAN_OP_NAME, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, @@ -3458,3 +3461,5 @@ static void destoryTableCountScanOperator(void* param) { taosArrayDestroy(pTableCountScanInfo->stbUidList); taosMemoryFreeClear(param); } + +// clang-format on From 76106165daf887be46463b78f49cc21a1d3c59f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 15:13:25 +0800 Subject: [PATCH 165/187] other: add some logs. --- source/libs/executor/src/scanoperator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index cea1e6361d..83a4f91034 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2090,7 +2090,9 @@ FETCH_NEXT_BLOCK: while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) { SSDataBlock* pRes = NULL; int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); + if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { + qDebug("failed to retrieve data from block, code:%s, rows:%"PRId64 " try next block in submit block", tstrerror(code), pRes->info.rows); continue; } From bcd8867fb2010892bdae2b3fa92edef8b4e60fd0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 15:27:14 +0800 Subject: [PATCH 166/187] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 83a4f91034..8151c459e5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1897,6 +1897,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pCreateTbRes, "recover createTbl"); return pInfo->pCreateTbRes; } + qDebug("stream recover scan get block, rows %" PRId64, pInfo->pRecoverRes->info.rows); printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; @@ -2089,10 +2090,13 @@ FETCH_NEXT_BLOCK: while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) { SSDataBlock* pRes = NULL; + int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); + qDebug("retrieve data from submit completed code:%s, rows:%" PRId64 " %s", + tstrerror(code), pRes->info.rows, id); if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { - qDebug("failed to retrieve data from block, code:%s, rows:%"PRId64 " try next block in submit block", tstrerror(code), pRes->info.rows); + qDebug("retrieve data failed, try next block in submit block, %s", id); continue; } @@ -2100,6 +2104,7 @@ FETCH_NEXT_BLOCK: if (pInfo->pCreateTbRes->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_RES; + qDebug("create table res exists, rows:%"PRId64" return from stream scan, %s", pInfo->pCreateTbRes->info.rows, id); return pInfo->pCreateTbRes; } From 2edba85efe930fe324c1c008799c1a7bfa8cadef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 15:27:44 +0800 Subject: [PATCH 167/187] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8151c459e5..0ed41aeea1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2092,8 +2092,8 @@ FETCH_NEXT_BLOCK: SSDataBlock* pRes = NULL; int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); - qDebug("retrieve data from submit completed code:%s, rows:%" PRId64 " %s", - tstrerror(code), pRes->info.rows, id); + qDebug("retrieve data from submit completed code:%s, rows:%" PRId64 " %s", tstrerror(code), pRes->info.rows, + id); if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { qDebug("retrieve data failed, try next block in submit block, %s", id); From 822f2673990766daf5d6403ecc054ea4c73e255f Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 15:36:36 +0800 Subject: [PATCH 168/187] enhance: install and uninstall rocksb.so in deb and rpm --- packaging/deb/DEBIAN/preinst | 1 + packaging/deb/DEBIAN/prerm | 1 + packaging/deb/makedeb.sh | 1 + packaging/rpm/tdengine.spec | 2 ++ 4 files changed, 5 insertions(+) diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index 904a946e20..d9951fc4d4 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -80,4 +80,5 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : +${csudo}rm -f ${install_main_dir}/driver/librocksdb.* || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 0d63115a04..80799ae0e5 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -40,6 +40,7 @@ else ${csudo}rm -f ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${log_link_dir} || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 9f49cf345a..624ccb0e3b 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -94,6 +94,7 @@ fi cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver +cp ${compile_dir}/build/lib/librocksdb.8 ${pkg_dir}${install_home_path}/driver [ -f ${compile_dir}/build/lib/${wslibfile} ] && cp ${compile_dir}/build/lib/${wslibfile} ${pkg_dir}${install_home_path}/driver ||: cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 52d5335003..31d597785e 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -174,6 +174,7 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f %{homepath}/driver/libtaos* || : +${csudo}rm -f %{homepath}/driver/librocksdb* || : #Scripts executed after installation %post @@ -219,6 +220,7 @@ if [ $1 -eq 0 ];then ${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : ${csudo}rm -f ${log_link_dir} || : ${csudo}rm -f ${data_link_dir} || : From 3d7defa881d8f8a146ffa6cba663d2ffe9a3677f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 15:43:38 +0800 Subject: [PATCH 169/187] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0ed41aeea1..df2062afe6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2113,6 +2113,8 @@ FETCH_NEXT_BLOCK: pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); + qDebug("%" PRId64 " rows in datablock, update res:%" PRId64 " %s", pBlockInfo->rows, + pInfo->pUpdateDataRes->info.rows, id); if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { break; } From 53696caabf16b91c1e621d6d1e11f1083fd1f9df Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 16:52:36 +0800 Subject: [PATCH 170/187] fix: rpm and deb librocksdb.so install and unintall --- packaging/deb/DEBIAN/preinst | 2 +- packaging/deb/DEBIAN/prerm | 2 +- packaging/deb/makedeb.sh | 3 ++- packaging/rpm/tdengine.spec | 2 ++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index d9951fc4d4..d6558d5b3b 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -80,5 +80,5 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : -${csudo}rm -f ${install_main_dir}/driver/librocksdb.* || : +[ -f ${install_main_dir}/driver/librocksdb.* ] && ${csudo}rm -f ${install_main_dir}/driver/librocksdb.* || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 80799ae0e5..8f8d472867 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -40,7 +40,7 @@ else ${csudo}rm -f ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : + [ -f ${lib_link_dir}/librocksdb.* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${log_link_dir} || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 624ccb0e3b..024c69deb1 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -31,6 +31,7 @@ cd ${pkg_dir} libfile="libtaos.so.${tdengine_ver}" wslibfile="libtaosws.so" +rocksdblib="librocksdb.so.8" # create install dir install_home_path="/usr/local/taos" @@ -94,7 +95,7 @@ fi cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver -cp ${compile_dir}/build/lib/librocksdb.8 ${pkg_dir}${install_home_path}/driver +[ -f ${compile_dir}/build/lib/${rocksdblib} ] && cp ${compile_dir}/build/lib/${rocksdblib} ${pkg_dir}${install_home_path}/driver ||: [ -f ${compile_dir}/build/lib/${wslibfile} ] && cp ${compile_dir}/build/lib/${wslibfile} ${pkg_dir}${install_home_path}/driver ||: cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 31d597785e..2b056c376a 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -45,6 +45,7 @@ echo buildroot: %{buildroot} libfile="libtaos.so.%{_version}" wslibfile="libtaosws.so" +rocksdblib="librocksdb.so.8" # create install path, and cp file mkdir -p %{buildroot}%{homepath}/bin @@ -92,6 +93,7 @@ if [ -f %{_compiledir}/build/bin/taosadapter ]; then fi cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver [ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile} %{buildroot}%{homepath}/driver ||: +[ -f %{_compiledir}/build/lib/${rocksdblib} ] && cp %{_compiledir}/build/lib/${rocksdblib} %{buildroot}%{homepath}/driver ||: cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include From 9f8f5374aaecd3f07f461240e2bc42ce35a56a8f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 16:56:20 +0800 Subject: [PATCH 171/187] fix: num_of_records_per_req item in json config need less than 32768. current = 100000 --- tests/system-test/1-insert/manyVgroups.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index 3b0fa96b08..8c6f39cf96 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -11,7 +11,7 @@ "confirm_parameter_prompt": "no", "insert_interval": 0, "interlace_rows": 0, - "num_of_records_per_req": 100000, + "num_of_records_per_req": 10000, "databases": [ { "dbinfo": { @@ -73,4 +73,4 @@ ] } ] -} \ No newline at end of file +} From 27d8870e182a6f31a4590869f3a7ec3166e29fd2 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Fri, 2 Jun 2023 17:09:03 +0800 Subject: [PATCH 172/187] fix: no data after seek --- source/client/src/clientTmq.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ccb09e3584..e1b2b9c48b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1868,7 +1868,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { // update the local offset value only for the returned values, only when the local offset is NOT updated // by tmq_offset_seek function if (!pVg->seekUpdated) { + tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId); pVg->offsetInfo.currentOffset = pDataRsp->rspOffset; + } else { + tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId); } // update the status @@ -1952,8 +1955,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { return NULL; } - if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate - pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; + // update the local offset value only for the returned values, only when the local offset is NOT updated + // by tmq_offset_seek function + if (!pVg->seekUpdated) { + if(pollRspWrapper->taosxRsp.rspOffset.type != 0) { // if offset is validate + tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId); + pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; + } + } else { + tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId); } atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); From 163f5abc4ce1841a6b5f0dcde8a9505c22178af0 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Fri, 2 Jun 2023 17:22:49 +0800 Subject: [PATCH 173/187] fix: no data after seek --- source/client/src/clientTmq.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ccb09e3584..e1b2b9c48b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1868,7 +1868,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { // update the local offset value only for the returned values, only when the local offset is NOT updated // by tmq_offset_seek function if (!pVg->seekUpdated) { + tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId); pVg->offsetInfo.currentOffset = pDataRsp->rspOffset; + } else { + tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId); } // update the status @@ -1952,8 +1955,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { return NULL; } - if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate - pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; + // update the local offset value only for the returned values, only when the local offset is NOT updated + // by tmq_offset_seek function + if (!pVg->seekUpdated) { + if(pollRspWrapper->taosxRsp.rspOffset.type != 0) { // if offset is validate + tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId); + pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; + } + } else { + tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId); } atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); From 449051cda67aff90ccb2e690b5521025803b78b0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 10:05:32 +0000 Subject: [PATCH 174/187] avoid mem leak --- source/libs/stream/src/streamBackendRocksdb.c | 4 ++-- source/libs/stream/src/tstreamFileState.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index b8af05c86d..30a7cefc9c 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1056,7 +1056,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa rocksdb_readoptions_t* opts = pState->pTdbState->readOpts; \ size_t len = 0; \ char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL) { \ + if (val == NULL || len == 0) { \ if (err == NULL) { \ qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ funcname); \ @@ -1069,7 +1069,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa } else { \ char* p = NULL; \ int32_t len = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (len < 0) { \ + if (len <= 0) { \ qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ funcname); \ code = -1; \ diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 54832b042a..7ea1a41925 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -416,7 +416,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { int32_t len = 0; memcpy(buf, taskKey, strlen(taskKey)); code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len); - if (code != 0) { + if (code != 0 || len == 0 || val == NULL) { return TSDB_CODE_FAILED; } sscanf(val, "%" PRId64 "", &maxCheckPointId); From 8d3b3dbedc360a8d62bd08ff3a15d74e62035a3e Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 18:11:56 +0800 Subject: [PATCH 175/187] enchance: main geos use static link, remove and install librocksdb --- cmake/cmake.options | 2 +- cmake/cmake.platform | 10 ---------- packaging/deb/DEBIAN/preinst | 1 + packaging/deb/DEBIAN/prerm | 1 + packaging/deb/makedeb.sh | 2 ++ packaging/rpm/tdengine.spec | 5 ++++- packaging/tools/install.sh | 9 +++++++++ packaging/tools/makepkg.sh | 3 +++ packaging/tools/post.sh | 6 ++++++ packaging/tools/remove.sh | 3 +++ 10 files changed, 30 insertions(+), 12 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index b00ae14715..555b72cbdf 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -80,7 +80,7 @@ ENDIF () option( BUILD_GEOS "If build geos on Windows" - OFF + ON ) option( diff --git a/cmake/cmake.platform b/cmake/cmake.platform index f9faf7316c..ba747c6134 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -56,17 +56,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(TD_DARWIN TRUE) SET(OSTYPE "macOS") - execute_process(COMMAND geos-config --cflags OUTPUT_VARIABLE GEOS_CFLAGS) - execute_process(COMMAND geos-config --ldflags OUTPUT_VARIABLE GEOS_LDFLAGS) - string(SUBSTRING ${GEOS_CFLAGS} 2 -1 GEOS_CFLAGS) - string(REGEX REPLACE "\n" "" GEOS_CFLAGS ${GEOS_CFLAGS}) - string(SUBSTRING ${GEOS_LDFLAGS} 2 -1 GEOS_LDFLAGS) - string(REGEX REPLACE "\n" "" GEOS_LDFLAGS ${GEOS_LDFLAGS}) - MESSAGE("GEOS_CFLAGS "${GEOS_CFLAGS}) - MESSAGE("GEOS_LDFLAGS "${GEOS_LDFLAGS}) ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") - INCLUDE_DIRECTORIES(${GEOS_CFLAGS}) - LINK_DIRECTORIES(${GEOS_LDFLAGS}) IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") MESSAGE("Current system arch is arm64") diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index 904a946e20..f2f98952db 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -80,4 +80,5 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : +[ -f ${install_main_dir}/driver/librocksdb.* ] && ${csudo}rm -f ${install_main_dir}/driver/librocksdb.* || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 0d63115a04..7cee258606 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -40,6 +40,7 @@ else ${csudo}rm -f ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + [ -f ${lib_link_dir}/librocksdb.* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${log_link_dir} || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 9f49cf345a..d5dbd9e4b8 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -31,6 +31,7 @@ cd ${pkg_dir} libfile="libtaos.so.${tdengine_ver}" wslibfile="libtaosws.so" +rocksdblib="librocksdb.so.8" # create install dir install_home_path="/usr/local/taos" @@ -94,6 +95,7 @@ fi cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver +[ -f ${compile_dir}/build/lib/${rocksdblib} ] && cp ${compile_dir}/build/lib/${rocksdblib} ${pkg_dir}${install_home_path}/driver ||: [ -f ${compile_dir}/build/lib/${wslibfile} ] && cp ${compile_dir}/build/lib/${wslibfile} ${pkg_dir}${install_home_path}/driver ||: cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 52d5335003..144cb0cfb2 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -45,6 +45,7 @@ echo buildroot: %{buildroot} libfile="libtaos.so.%{_version}" wslibfile="libtaosws.so" +rocksdblib="librocksdb.so.8" # create install path, and cp file mkdir -p %{buildroot}%{homepath}/bin @@ -92,6 +93,7 @@ if [ -f %{_compiledir}/build/bin/taosadapter ]; then fi cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver [ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile} %{buildroot}%{homepath}/driver ||: +[ -f %{_compiledir}/build/lib/${rocksdblib} ] && cp %{_compiledir}/build/lib/${rocksdblib} %{buildroot}%{homepath}/driver ||: cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include @@ -174,6 +176,7 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f %{homepath}/driver/libtaos* || : +${csudo}rm -f %{homepath}/driver/librocksdb* || : #Scripts executed after installation %post @@ -219,7 +222,7 @@ if [ $1 -eq 0 ];then ${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : ${csudo}rm -f ${log_link_dir} || : ${csudo}rm -f ${data_link_dir} || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 1b47b10520..1501b15196 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -250,18 +250,27 @@ function install_lib() { # Remove links ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : + ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : #${csudo}rm -rf ${v15_java_app_dir} || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib_link_dir}/librocksdb.so.8 + ${csudo}ln -sf ${lib_link_dir}/librocksdb.so.8 ${lib_link_dir}/librocksdb.so + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib64_link_dir}/librocksdb.so.8 || : + ${csudo}ln -sf ${lib64_link_dir}/librocksdb.so.8 ${lib64_link_dir}/librocksdb.so || : + [ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index b0537e8bcf..bf35734da3 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -111,9 +111,11 @@ fi if [ "$osType" == "Darwin" ]; then lib_files="${build_dir}/lib/libtaos.${version}.dylib" wslib_files="${build_dir}/lib/libtaosws.dylib" + rocksdb_lib_files="${build_dir}/lib/librocksdb.so.8.1.1" else lib_files="${build_dir}/lib/libtaos.so.${version}" wslib_files="${build_dir}/lib/libtaosws.so" + rocksdb_lib_files="${build_dir}/lib/librocksdb.so.8.1.1" fi header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" @@ -336,6 +338,7 @@ fi # Copy driver mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : +[ -f ${rocksdb_lib_files} ] && cp ${rocksdb_lib_files} ${install_dir}/driver || : # Copy connector if [ "$verMode" == "cluster" ]; then diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index fc392c9684..1b5eb45f66 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -203,9 +203,14 @@ function install_lib() { ${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || : + #rocksdb + [ -f ${lib_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb* || : + [ -f ${lib64_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib64_link_dir}/librocksdb* || : + [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : + ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 @@ -214,6 +219,7 @@ function install_lib() { if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.${lib_file_ext} ]]; then ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 + ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib64_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} 2>>${install_log_path} fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 6c671473bf..a17b29983c 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -142,11 +142,14 @@ function clean_local_bin() { function clean_lib() { # Remove link ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : + } function clean_header() { From 17f2c3f88249b07ed79c5888bdf505c1322d9f20 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 2 Jun 2023 16:51:44 +0800 Subject: [PATCH 176/187] reset wind info --- source/libs/executor/inc/executorInt.h | 1 + source/libs/executor/src/timewindowoperator.c | 25 +++++++++++++++++++ source/libs/stream/src/tstreamFileState.c | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 79391bc1c5..ffc63a22a8 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -456,6 +456,7 @@ typedef struct SStreamIntervalOperatorInfo { SSHashObj* pUpdatedMap; int64_t dataVersion; SStateStore statestore; + bool recvGetAll; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index fc08f827a5..3a83472079 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2439,6 +2439,15 @@ static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) { return 0; } +static void resetUnCloseWinInfo(SSHashObj* winMap) { + void* pIte = NULL; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) { + SRowBuffPos* pPos = *(SRowBuffPos**)pIte; + pPos->beUsed = true; + } +} + static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2472,6 +2481,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable); + } + setOperatorCompleted(pOperator); if (!IS_FINAL_OP(pInfo)) { clearFunctionContext(&pOperator->exprSupp); @@ -2565,6 +2579,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { + pInfo->recvGetAll = true; getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { @@ -2773,6 +2788,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); pInfo->dataVersion = 0; pInfo->statestore = pTaskInfo->storageAPI.stateStore; + pInfo->recvGetAll = false; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -4751,6 +4767,12 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes; } + + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable); + } + setOperatorCompleted(pOperator); if (pInfo->twAggSup.maxTs > 0 && pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { @@ -4790,6 +4812,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_GET_ALL) { qDebug("===stream===single interval recv|block type STREAM_GET_ALL"); + pInfo->recvGetAll = true; getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { @@ -4960,6 +4983,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); pInfo->statestore = pTaskInfo->storageAPI.stateStore; + pInfo->recvGetAll = false; + initIntervalDownStream(downstream, pPhyNode->type, pInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index f531f65565..e13499d87f 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -137,7 +137,7 @@ void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) { SListNode* pNode = NULL; while ((pNode = tdListNext(&iter)) != NULL) { SRowBuffPos* pPos = *(SRowBuffPos**)(pNode->data); - if (all || (pFileState->getTs(pPos->pKey) < ts)) { + if (all || (pFileState->getTs(pPos->pKey) < ts && !pPos->beUsed)) { ASSERT(pPos->pRowBuff != NULL); tdListAppend(pFileState->freeBuffs, &(pPos->pRowBuff)); pPos->pRowBuff = NULL; From 6ee68331b319aefe7578a0051285ad4a7069588f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 11:54:36 +0000 Subject: [PATCH 177/187] update DEFAULT_BUFF --- source/libs/stream/src/streamBackendRocksdb.c | 7 +++++-- source/libs/stream/src/streamUpdate.c | 12 ++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 30a7cefc9c..e07198f6ab 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -87,8 +87,6 @@ void* streamBackendInit(const char* path) { pHandle->cfInst = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); rocksdb_env_t* env = rocksdb_create_default_env(); // rocksdb_envoptions_create(); - rocksdb_env_set_low_priority_background_threads(env, 4); - rocksdb_env_set_high_priority_background_threads(env, 2); rocksdb_cache_t* cache = rocksdb_cache_create_lru(64 << 20); @@ -577,6 +575,11 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { int64_t now = taosGetTimestampMs(); p = taosDecodeFixedI64(p, &key.unixTimestamp); p = taosDecodeFixedI32(p, &key.len); + if (vlen != sizeof(int64_t) + sizeof(int32_t) + key.len) { + *dest = NULL; + return -1; + } + if (key.len == 0) { key.data = NULL; } else { diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index fff666ec9f..85be120dbd 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -20,9 +20,9 @@ #include "ttime.h" #define DEFAULT_FALSE_POSITIVE 0.01 -#define DEFAULT_BUCKET_SIZE 1310720 -#define DEFAULT_MAP_CAPACITY 1310720 -#define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 10) +#define DEFAULT_BUCKET_SIZE 131072 +#define DEFAULT_MAP_CAPACITY 131072 +#define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 100) #define ROWS_PER_MILLISECOND 1 #define MAX_NUM_SCALABLE_BF 100000 #define MIN_NUM_SCALABLE_BF 10 @@ -44,8 +44,8 @@ static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { } } -static void clearItemHelper(void* p) { - SScalableBf** pBf = p; +static void clearItemHelper(void *p) { + SScalableBf **pBf = p; tScalableBfDestroy(*pBf); } @@ -274,7 +274,7 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { } int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) { - if(!pInfo) { + if (!pInfo) { return 0; } From e48e485b6c71165a1546cbdf4ecef37b9e599225 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 11:56:24 +0000 Subject: [PATCH 178/187] update default buffer --- source/libs/stream/src/streamBackendRocksdb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index e07198f6ab..5be6d1b003 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -572,11 +572,10 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { *dest = NULL; return -1; } - int64_t now = taosGetTimestampMs(); p = taosDecodeFixedI64(p, &key.unixTimestamp); p = taosDecodeFixedI32(p, &key.len); if (vlen != sizeof(int64_t) + sizeof(int32_t) + key.len) { - *dest = NULL; + if (dest != NULL) *dest = NULL; return -1; } @@ -587,6 +586,7 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { } if (ttl != NULL) { + int64_t now = taosGetTimestampMs(); *ttl = key.unixTimestamp == 0 ? 0 : key.unixTimestamp - now; } if (dest != NULL) { From 3ec700e0f7c20d036a87c50d140c08dd17b2edda Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 22:25:38 +0800 Subject: [PATCH 179/187] fix mem problem --- source/libs/stream/src/streamBackendRocksdb.c | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5be6d1b003..6309cb9df2 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -576,6 +576,7 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { p = taosDecodeFixedI32(p, &key.len); if (vlen != sizeof(int64_t) + sizeof(int32_t) + key.len) { if (dest != NULL) *dest = NULL; + qError("vlen: %d, read len: %d", vlen, key.len); return -1; } @@ -1008,35 +1009,35 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[idx]); } -#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamGetInit(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = \ - ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ - char* ttlV = NULL; \ - int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ - rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ - if (err != NULL) { \ - taosMemoryFree(err); \ - qError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ - code = -1; \ - } else { \ - qTrace("streamState str:%s succ to write to %s, valLen:%d", toString, funcname, vLen); \ - } \ - taosMemoryFree(ttlV); \ +#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamGetInit(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = \ + ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = pState->pTdbState->rocksdb; \ + rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ + char* ttlV = NULL; \ + int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ + rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ + if (err != NULL) { \ + taosMemoryFree(err); \ + qError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ + code = -1; \ + } else { \ + qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d", toString, funcname, vLen, ttlVLen); \ + } \ + taosMemoryFree(ttlV); \ } while (0); #define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ @@ -1071,17 +1072,17 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa code = -1; \ } else { \ char* p = NULL; \ - int32_t len = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (len <= 0) { \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ funcname); \ code = -1; \ } else { \ qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, pState->pTdbState->idstr, funcname, \ - len); \ + tlen); \ } \ taosMemoryFree(val); \ - if (vLen != NULL) *vLen = len; \ + if (vLen != NULL) *vLen = tlen; \ } \ if (code == 0) \ qDebug("streamState str: %s succ to read from %s_%s", toString, pState->pTdbState->idstr, funcname); \ From b5b4cd2a05b686ab0de277d3354fad3f62c04f86 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 23:15:27 +0800 Subject: [PATCH 180/187] fix(query): load del info with upper version limitation. --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbDataIter.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 17 +++++++++++++---- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 12 ++++++++---- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++-- source/libs/wal/src/walRead.c | 2 +- 8 files changed, 28 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index faf550ab75..828e0e7e3d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -297,7 +297,7 @@ int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 6a97ea89b3..d6c19ee18e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1454,7 +1454,7 @@ static int32_t getTableDelDataFromDelIdx(SDelFReader *pDelReader, SDelIdx *pDelI int32_t code = 0; if (pDelIdx) { - code = tsdbReadDelData(pDelReader, pDelIdx, aDelData); + code = tsdbReadDelData(pDelReader, pDelIdx, aDelData, INT64_MAX); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d15f848cfd..ba74613bd9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -266,7 +266,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel suid = pDelIdx->suid; uid = pDelIdx->uid; - code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData); + code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); } else { taosArrayClear(pCommitter->aDelData); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataIter.c b/source/dnode/vnode/src/tsdb/tsdbDataIter.c index e27aec5b1b..701ad56181 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataIter.c @@ -412,7 +412,7 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* } } - code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData); + code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); pIter->delInfo.suid = pDelIdx->suid; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9cae4ad5aa..1286c2f7c4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2967,7 +2967,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SDelIdx* pIdx = taosArraySearch(pReader->pDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData); + code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData, pReader->verRange.maxVer); } if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -2978,7 +2978,10 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* if (pMemTbData != NULL) { p = pMemTbData->pHead; while (p) { - taosArrayPush(pDelData, p); + if (p->version <= pReader->verRange.maxVer) { + taosArrayPush(pDelData, p); + } + p = p->pNext; } } @@ -2986,7 +2989,9 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* if (piMemTbData != NULL) { p = piMemTbData->pHead; while (p) { - taosArrayPush(pDelData, p); + if (p->version <= pReader->verRange.maxVer) { + taosArrayPush(pDelData, p); + } p = p->pNext; } } @@ -4558,7 +4563,11 @@ int32_t tsdbReaderOpen(void* pVnode, SQueryTableDataCond* pCond, void* pTableLis pReader->pIgnoreTables = pIgnoreTables; - tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); + tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64 + " in this query %s", + pReader, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->verRange.minVer, + pReader->verRange.maxVer, pReader->idStr); + return code; _err: diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index b25ab393da..e4b8a57291 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1488,7 +1488,7 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { return code; } -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t code = 0; int64_t offset = pDelIdx->offset; int64_t size = pDelIdx->size; @@ -1510,11 +1510,15 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData SDelData delData; n += tGetDelData(pReader->aBuf[0] + n, &delData); - if (taosArrayPush(aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + if (delData.version > maxVer) { + continue; } + if (taosArrayPush(aDelData, &delData) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } + ASSERT(n == size); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index dfea125cc1..f7362b366f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1166,7 +1166,7 @@ static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* int32_t c = tTABLEIDCmprFn(pDelIdx, &pWriter->tbid); if (c < 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData); + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); SDelIdx* pDelIdxNew = taosArrayReserve(pWriter->aDelIdx, 1); @@ -1183,7 +1183,7 @@ static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pWriter->pTIter->tIter.iDelIdx++; } else if (c == 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); pWriter->pTIter->tIter.iDelIdx++; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 0eb3d6ef09..1223e3756c 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -333,7 +333,7 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { return -1; } - wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); + wDebug("vgId:%d, index:%" PRId64 " is fetched, type:%d, cursor advance", pReader->pWal->cfg.vgId, ver, pReader->pHead->head.msgType); pReader->curVersion = ver + 1; return 0; } From becacbc84b63a181758d16a0b033a89003d240b2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 23:36:47 +0800 Subject: [PATCH 181/187] fix(query): fix syntax error. --- source/dnode/vnode/src/inc/tsdb.h | 3 ++- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbDataIter.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++-- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 828e0e7e3d..9df95a379a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -297,7 +297,8 @@ int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer); +int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer); +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d6c19ee18e..c659c8f4a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1454,7 +1454,7 @@ static int32_t getTableDelDataFromDelIdx(SDelFReader *pDelReader, SDelIdx *pDelI int32_t code = 0; if (pDelIdx) { - code = tsdbReadDelData(pDelReader, pDelIdx, aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pDelReader, pDelIdx, aDelData, INT64_MAX); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ba74613bd9..b440d51883 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -266,7 +266,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel suid = pDelIdx->suid; uid = pDelIdx->uid; - code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); } else { taosArrayClear(pCommitter->aDelData); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataIter.c b/source/dnode/vnode/src/tsdb/tsdbDataIter.c index 701ad56181..8215c1ac29 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataIter.c @@ -412,7 +412,7 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* } } - code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); pIter->delInfo.suid = pDelIdx->suid; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1286c2f7c4..2500015ec1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2967,7 +2967,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SDelIdx* pIdx = taosArraySearch(pReader->pDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData, pReader->verRange.maxVer); + code = tsdbReadDelDatav1(pReader->pDelFReader, pIdx, pDelData, pReader->verRange.maxVer); } if (code != TSDB_CODE_SUCCESS) { goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index e4b8a57291..fb899fa1ce 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1488,7 +1488,7 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { return code; } -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { +int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t code = 0; int64_t offset = pDelIdx->offset; int64_t size = pDelIdx->size; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index f7362b366f..b5ca716701 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1166,7 +1166,7 @@ static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* int32_t c = tTABLEIDCmprFn(pDelIdx, &pWriter->tbid); if (c < 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); SDelIdx* pDelIdxNew = taosArrayReserve(pWriter->aDelIdx, 1); @@ -1183,7 +1183,7 @@ static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pWriter->pTIter->tIter.iDelIdx++; } else if (c == 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); pWriter->pTIter->tIter.iDelIdx++; From 5063c175f4ff62ceb210d5424d1d7bda5681d1b9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 23:47:14 +0800 Subject: [PATCH 182/187] fix mem problem --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- source/libs/stream/src/tstreamFileState.c | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 6309cb9df2..b3995f020b 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -574,7 +574,7 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { } p = taosDecodeFixedI64(p, &key.unixTimestamp); p = taosDecodeFixedI32(p, &key.len); - if (vlen != sizeof(int64_t) + sizeof(int32_t) + key.len) { + if (vlen != (sizeof(int64_t) + sizeof(int32_t) + key.len)) { if (dest != NULL) *dest = NULL; qError("vlen: %d, read len: %d", vlen, key.len); return -1; diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 7ea1a41925..757b1de6b6 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -419,7 +419,9 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (code != 0 || len == 0 || val == NULL) { return TSDB_CODE_FAILED; } - sscanf(val, "%" PRId64 "", &maxCheckPointId); + memcpy(val, buf, len); + buf[len] = 0; + maxCheckPointId = atol((char*)buf); taosMemoryFree(val); } for (int64_t i = maxCheckPointId; i > 0; i--) { @@ -431,16 +433,17 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (code != 0) { return TSDB_CODE_FAILED; } + memcpy(val, buf, len); + buf[len] = 0; + taosMemoryFree(val); + TSKEY ts; - sscanf(val, "%" PRId64 "", &ts); + ts = atol((char*)buf); if (ts < mark) { // statekey winkey.ts < mark forceRemoveCheckpoint(pFileState, i); - taosMemoryFreeClear(val); break; - } else { } - taosMemoryFree(val); } return code; } From 4bb64292498390fd17abb5fc5e141a1c2b34920c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Jun 2023 00:20:59 +0800 Subject: [PATCH 183/187] fix(query): fix syntax error. --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index fb899fa1ce..4b677533e7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1488,6 +1488,10 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { return code; } +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { + return tsdbReadDelDatav1(pReader, pDelIdx, aDelData, INT64_MAX); +} + int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t code = 0; int64_t offset = pDelIdx->offset; From 81fe174aa26b0dd75ed9f805698edf919ed34ef3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 3 Jun 2023 08:19:59 +0800 Subject: [PATCH 184/187] fix mem leak --- source/libs/stream/src/streamExec.c | 39 ++++++++++++++++------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 716b939e5f..1a78607770 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -16,11 +16,11 @@ #include "streamInc.h" // maximum allowed processed block batches. One block may include several submit blocks -#define MAX_STREAM_EXEC_BATCH_NUM 32 -#define MIN_STREAM_EXEC_BATCH_NUM 8 -#define MAX_STREAM_RESULT_DUMP_THRESHOLD 100 +#define MAX_STREAM_EXEC_BATCH_NUM 32 +#define MIN_STREAM_EXEC_BATCH_NUM 8 +#define MAX_STREAM_RESULT_DUMP_THRESHOLD 100 -static int32_t updateCheckPointInfo (SStreamTask* pTask); +static int32_t updateCheckPointInfo(SStreamTask* pTask); bool streamTaskShouldStop(const SStreamStatus* pStatus) { int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus); @@ -48,10 +48,11 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return -1; } - qDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, size/1048576.0); + qDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, + size / 1048576.0); code = streamTaskOutputResultBlock(pTask, pStreamBlocks); - if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { // back pressure and record position + if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { // back pressure and record position destroyStreamDataBlock(pStreamBlocks); return -1; } @@ -65,7 +66,8 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return TSDB_CODE_SUCCESS; } -static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { +static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, + int32_t* totalBlocks) { int32_t code = TSDB_CODE_SUCCESS; void* pExecutor = pTask->exec.pExecutor; @@ -82,7 +84,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i } if (streamTaskShouldStop(&pTask->status)) { - taosArrayDestroy(pRes); // memory leak + taosArrayDestroy(pRes); // memory leak return 0; } @@ -101,7 +103,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SSDataBlock block = {0}; - const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*) pItem; + const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); @@ -153,7 +155,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i ASSERT(numOfBlocks == taosArrayGetSize(pRes)); code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); } else { - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); } return code; @@ -286,7 +288,7 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { } #endif -int32_t updateCheckPointInfo (SStreamTask* pTask) { +int32_t updateCheckPointInfo(SStreamTask* pTask) { int64_t ckId = 0; int64_t dataVer = 0; qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId); @@ -294,7 +296,8 @@ int32_t updateCheckPointInfo (SStreamTask* pTask) { SCheckpointInfo* pCkInfo = &pTask->chkInfo; if (ckId > pCkInfo->id) { // save it since the checkpoint is updated qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64 - ", checkPoint id:%" PRId64 " -> %" PRId64, pTask->id.idStr, pCkInfo->version, dataVer, pCkInfo->id, ckId); + ", checkPoint id:%" PRId64 " -> %" PRId64, + pTask->id.idStr, pCkInfo->version, dataVer, pCkInfo->id, ckId); pTask->chkInfo = (SCheckpointInfo){.version = dataVer, .id = ckId, .currentVer = pCkInfo->currentVer}; @@ -417,14 +420,15 @@ int32_t streamExecForAll(SStreamTask* pTask) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); - qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, - pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); + qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, + pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)pInput; SArray* pBlockList = pBlock->blocks; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); + qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, + pBlock->sourceVer); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput; @@ -445,8 +449,9 @@ int32_t streamExecForAll(SStreamTask* pTask) { int32_t totalBlocks = 0; streamTaskExecImpl(pTask, pInput, &resSize, &totalBlocks); - double el = (taosGetTimestampMs() - st) / 1000.0; - qDebug("s-task:%s exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", pTask->id.idStr, el, resSize / 1048576.0, totalBlocks); + double el = (taosGetTimestampMs() - st) / 1000.0; + qDebug("s-task:%s exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", pTask->id.idStr, el, + resSize / 1048576.0, totalBlocks); streamFreeQitem(pInput); } From 7244ab1f6bfd28c1d5d2a9cbc4f4442a8bb50c94 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 3 Jun 2023 08:28:21 +0800 Subject: [PATCH 185/187] fix mem leak --- source/libs/stream/src/streamExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 1a78607770..aa2c8050de 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -84,7 +84,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i } if (streamTaskShouldStop(&pTask->status)) { - taosArrayDestroy(pRes); // memory leak + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return 0; } From 9e0392ff59a1db9bcee81c01dc7b30cc2fdda8ff Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 3 Jun 2023 09:17:05 +0800 Subject: [PATCH 186/187] fix mem leak --- source/libs/stream/src/streamExec.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index aa2c8050de..49b35ce8db 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -101,8 +101,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; - + SSDataBlock block = {0}; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); From 0f69aa0b7a65edb0c2f19b42487693d0e812dbd3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Jun 2023 17:24:17 +0800 Subject: [PATCH 187/187] other: set the tags for 3.0 branch. --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9bbda8309f..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE