From 0d9d2ea293c5b0679be29bdafa7f2ed5eab1188c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Feb 2023 18:31:32 +0800 Subject: [PATCH 001/238] fix(query): do some internal refactor, and fix the bug when calender time duration exists in sliding and offset. --- include/common/ttime.h | 2 +- include/libs/executor/executor.h | 4 +- source/common/src/ttime.c | 114 +++++++++++------- source/libs/executor/inc/executil.h | 3 + source/libs/executor/inc/executorimpl.h | 1 - source/libs/executor/src/executil.c | 37 +++++- source/libs/executor/src/executorimpl.c | 6 +- source/libs/executor/src/filloperator.c | 8 +- source/libs/executor/src/scanoperator.c | 37 +----- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 47 +------- source/libs/planner/src/planOptimizer.c | 4 +- 12 files changed, 129 insertions(+), 136 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index eaf44c2771..69a071d59f 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -74,7 +74,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 63e1c556de..382837951a 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -178,7 +178,9 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key); /** * return the scan info, in the form of tuple of two items, including table uid and current timestamp * @param tinfo diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 559ffd2aaf..bbb58cc63d 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -82,6 +82,7 @@ static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_ static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); +static int32_t parseTimezone(char* str, int64_t* tzOffset); static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = { parseLocaltime, parseLocaltimeDst}; @@ -708,21 +709,19 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati return getDuration(*duration, *unit, duration, timePrecision); } +#define IS_CALENDAR_TIME_DURATION(_d) (((_d) == 'n') || ((_d) == 'y')) + int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { return t; } - if (unit != 'n' && unit != 'y') { + if (!IS_CALENDAR_TIME_DURATION(unit)) { return t + duration; } // The following code handles the y/n time duration - int64_t numOfMonth = duration; - if (unit == 'y') { - numOfMonth *= 12; - } - + int64_t numOfMonth = (unit == 'y')? duration*12:duration; int64_t fraction = t % TSDB_TICK_PER_SECOND(precision); struct tm tm; @@ -764,13 +763,16 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char return (emon - smon) / (int32_t)interval; } -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (pInterval->sliding == 0 && pInterval->interval == 0) { - return t; + return ts; } - int64_t start = t; - if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') { + int64_t start = ts; + int32_t precision = pInterval->precision; + + if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { + start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; @@ -792,44 +794,72 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start = (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision)); } else { - int64_t delta = t - pInterval->interval; - int32_t factor = (delta >= 0) ? 1 : -1; + if (IS_CALENDAR_TIME_DURATION(pInterval->intervalUnit)) { + int64_t news = (ts / pInterval->sliding) * pInterval->sliding; + ASSERT(news <= ts); - start = (delta / pInterval->sliding + factor) * pInterval->sliding; + if (news <= ts) { + int64_t prev = news; + int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; - if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { - /* - * here we revised the start time of day according to the local time zone, - * but in case of DST, the start time of one day need to be dynamically decided. - */ - // todo refactor to extract function that is available for Linux/Windows/Mac platform -#if defined(WINDOWS) && _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char** tzname = _tzname; -#endif + if (newe < ts) { // move towards the greater endpoint + while(newe < ts && news < ts) { + news += pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } - start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); - } - - int64_t end = 0; - - // not enough time range - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - while (end < t) { // move forward to the correct time window - start += pInterval->sliding; - - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = start + pInterval->interval - 1; + prev = news; } else { - end = INT64_MAX; - break; + while (newe >= ts) { + prev = news; + news -= pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } } + + return prev; } } else { - end = INT64_MAX; + int64_t delta = ts - pInterval->interval; + int32_t factor = (delta >= 0) ? 1 : -1; + + start = (delta / pInterval->sliding + factor) * pInterval->sliding; + + if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { + /* + * here we revised the start time of day according to the local time zone, + * but in case of DST, the start time of one day need to be dynamically decided. + */ + // todo refactor to extract function that is available for Linux/Windows/Mac platform +#if defined(WINDOWS) && _MSC_VER >= 1900 + // see + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char** tzname = _tzname; +#endif + + start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); + } + + int64_t end = 0; + + // not enough time range + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + while (end < ts) { // move forward to the correct time window + start += pInterval->sliding; + + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = start + pInterval->interval - 1; + } else { + end = INT64_MAX; + break; + } + } + } else { + end = INT64_MAX; + } } } @@ -842,7 +872,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; int64_t newEnd = end; - while (newEnd >= t) { + while (newEnd >= ts) { end = newEnd; newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f99c7de93d..20f43deaa3 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -164,4 +164,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, void printDataBlock(SSDataBlock* pBlock, const char* flag); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 999a7965fb..1850cd007f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -854,7 +854,6 @@ int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResul SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 040e67713d..53947e85e7 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1686,12 +1686,12 @@ int32_t convertFillType(int32_t mode) { return type; } -static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { if (ascQuery) { - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); int64_t key = w->skey; while (key < ts) { // moving towards end @@ -1708,7 +1708,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindo static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { STimeWindow w = {0}; - w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision); + w.skey = taosTimeTruncate(ts, pInterval); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return w; } @@ -1742,6 +1742,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI if (pRow) { w = pRow->win; } + // in case of typical time window, we can calculate time window directly. if (w.skey > ts || w.ekey < ts) { w = doCalculateTimeWindow(ts, pInterval); @@ -1756,6 +1757,34 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI return w; } +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (pInterval->sliding != 'n' && pInterval->sliding != 'y') { + tw->skey += pInterval->sliding * factor; + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return; + } + + // convert key to second + int64_t key = convertTimePrecision(tw->skey, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + int64_t duration = pInterval->sliding; + if (pInterval->slidingUnit == 'y') { + duration *= 12; + } + + struct tm tm; + time_t t = (time_t) key; + taosLocalTime(&t, &tm); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + duration * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); + + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; +} + bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { return (pLimitInfo->limit.limit != -1 || pLimitInfo->limit.offset != -1 || pLimitInfo->slimit.limit != -1 || pLimitInfo->slimit.offset != -1); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d28c3cfe58..464282613d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -637,15 +637,15 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code) ? tr void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; } ///////////////////////////////////////////////////////////////////////////////////////////// -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) { +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) { STimeWindow win = {0}; - win.skey = taosTimeTruncate(key, pInterval, precision); + win.skey = taosTimeTruncate(key, pInterval); /* * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; if (win.ekey < win.skey) { win.ekey = INT64_MAX; } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 16983cb507..3403c202b7 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -255,10 +255,10 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); + STimeWindow w = {0}; int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey); - w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order); + getInitialStartTimeWindow(pInterval, startKey, &w, order); pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); @@ -400,13 +400,13 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); return win.skey; } TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); return win.skey; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b556733254..c83c161326 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -67,39 +67,6 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - tw->ekey -= 1; -} - static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) { STimeWindow w = {0}; @@ -109,7 +76,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } if (order == TSDB_ORDER_ASC) { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey); ASSERT(w.ekey >= pBlockInfo->window.skey); if (w.ekey < pBlockInfo->window.ekey) { @@ -128,7 +95,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } } } else { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey); ASSERT(w.skey <= pBlockInfo->window.ekey); if (w.skey > pBlockInfo->window.skey) { diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 778281f9b4..ed786003d5 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -519,7 +519,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) pFillInfo->end = endKey; if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->interval.precision); + pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval); } pFillInfo->index = 0; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c5dc927bd1..15f803125c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -263,43 +263,6 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary return num; } -static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - tw->ekey -= 1; -} - -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - getNextTimeWindow(pInterval, pInterval->precision, order, tw); -} - void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { SqlFunctionCtx* pCtx = pSup->pCtx; @@ -455,7 +418,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; - getNextTimeWindow(pInterval, precision, order, pNext); + getNextTimeWindow(pInterval, pNext, order); // next time window is not in current block if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || @@ -500,7 +463,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, if (ascQuery && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -509,7 +472,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -1380,7 +1343,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa do { if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i])) { - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } uint64_t winGpId = pGpDatas[i]; @@ -1397,7 +1360,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); } } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); } while (win.ekey <= endTsCols[i]); } } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 90a7261074..27c1555666 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1247,8 +1247,8 @@ static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pW .sliding = pIndex->sliding, .slidingUnit = pIndex->slidingUnit, .precision = pScan->node.precision}; - return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval, pScan->node.precision)) && - (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval, pScan->node.precision)); + return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval)) && + (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval)); } return true; } From 9b90416bf32effdd2617124394cedeab9dd2b721 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Feb 2023 18:38:08 +0800 Subject: [PATCH 002/238] test: add unit test case. --- source/libs/executor/test/timewindowTest.cpp | 141 +++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 source/libs/executor/test/timewindowTest.cpp diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp new file mode 100644 index 0000000000..6635c820fa --- /dev/null +++ b/source/libs/executor/test/timewindowTest.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taos.h" +#include "thash.h" +#include "tsimplehash.h" +#include "executor.h" +#include "ttime.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +namespace { +SInterval createInterval(int64_t interval, int64_t sliding, int64_t offset, char intervalUnit, char slidingUnit, + char offsetUnit, int8_t precision) { + SInterval v = {0}; + v.interval = interval; + v.intervalUnit = intervalUnit; + v.sliding = sliding; + v.slidingUnit = slidingUnit; + v.offset = offset; + v.offsetUnit = offsetUnit; + v.precision = precision; + return v; +} + +void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) { + char buf[64] = {0}; + char bufs[64] = {0}; + char bufe[64] = {0}; + + taosFormatUtcTime(buf, tListLen(buf), ts, precision); + + taosFormatUtcTime(bufs, tListLen(bufs), pWindow->skey, precision); + taosFormatUtcTime(bufe, tListLen(bufe), pWindow->ekey, precision); + + printf("%s [%s - %s]\n", buf, bufs, bufe); +} +} // namespace + +TEST(testCase, timewindow_gen) { + // time window + osSetTimezone("UTC"); + + SInterval interval = + createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + + int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms) + + STimeWindow w = {0}; + getInitialStartTimeWindow(&interval, key, &w, true); + printTimeWindow(&w, interval.precision, key); + + getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, interval.precision, key); + + printf("---------------------------------------------------\n"); + SInterval monthInterval = + createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&monthInterval, key, &w, true); + printTimeWindow(&w, monthInterval.precision, key); + + getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, monthInterval.precision, key); + + printf("----------------------------------------------------------\n"); + SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&slidingInterval, key, &w, true); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + printf("----------------------------------------------------------\n"); + SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + int64_t k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true); + printTimeWindow(&w, calendar_interval_1n.precision, k1); + + printf("next\n"); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + + printf("----------------------------------------------------------\n"); + SInterval calendar_interval_1d_sliding_1n = createInterval(1*86400*100L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + +} + +#pragma GCC diagnostic pop \ No newline at end of file From 579b1b0a6c142c26b6f14261076d7ec54aee7bd1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 Feb 2023 15:12:19 +0800 Subject: [PATCH 003/238] fix(query): check correct unit. --- source/libs/executor/src/executil.c | 2 +- source/libs/executor/test/timewindowTest.cpp | 65 ++++++++++++-------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 53947e85e7..91f384ce6a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1759,7 +1759,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->sliding != 'n' && pInterval->sliding != 'y') { + if (pInterval->slidingUnit != 'n' && pInterval->slidingUnit != 'y') { tw->skey += pInterval->sliding * factor; tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return; diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index 6635c820fa..be5e217ed1 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -56,85 +56,102 @@ void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) { } // namespace TEST(testCase, timewindow_gen) { - // time window + // set correct time zone osSetTimezone("UTC"); - + int32_t precision = TSDB_TIME_PRECISION_MILLI; + SInterval interval = - createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', precision); int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms) STimeWindow w = {0}; getInitialStartTimeWindow(&interval, key, &w, true); - printTimeWindow(&w, interval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC); printf("next\n"); - printTimeWindow(&w, interval.precision, key); + printTimeWindow(&w, precision, key); printf("---------------------------------------------------\n"); SInterval monthInterval = createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI); getInitialStartTimeWindow(&monthInterval, key, &w, true); - printTimeWindow(&w, monthInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC); printf("next\n"); - printTimeWindow(&w, monthInterval.precision, key); + printTimeWindow(&w, precision, key); printf("----------------------------------------------------------\n"); SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); getInitialStartTimeWindow(&slidingInterval, key, &w, true); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); printf("next\n"); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); - printf("----------------------------------------------------------\n"); + printf("-----------------calendar_interval_1n_sliding_1d-------\n"); SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); int64_t k1 = 1664409600 * 1000L; getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true); - printTimeWindow(&w, calendar_interval_1n.precision, k1); + printTimeWindow(&w, precision, k1); printf("next\n"); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); + printf("----------------interval_1d_clendar_sliding_1n---------\n"); + SInterval interval_1d_calendar_sliding_1n = createInterval(1*86400*1000L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); - printf("----------------------------------------------------------\n"); - SInterval calendar_interval_1d_sliding_1n = createInterval(1*86400*100L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&interval_1d_calendar_sliding_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + printf("next time window:\n"); + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n"); + SInterval offset_1n = createInterval(1*86400*1000L, 1*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); } From a488f31ba98998cd338e9f21dd135999e78a332f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Feb 2023 09:52:11 +0800 Subject: [PATCH 004/238] refactor:do some internal refactor. --- source/common/src/ttime.c | 8 ++++---- source/libs/executor/test/timewindowTest.cpp | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index bbb58cc63d..de6a18f7c8 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -871,10 +871,10 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { // try to move current window to the left-hande-side, due to the offset effect. int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - int64_t newEnd = end; - while (newEnd >= ts) { - end = newEnd; - newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); + int64_t newe = end; + while (newe >= ts) { + end = newe; + newe = taosTimeAdd(newe, -pInterval->sliding, pInterval->slidingUnit, precision); } start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index be5e217ed1..2894c66587 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -151,7 +151,10 @@ TEST(testCase, timewindow_gen) { printTimeWindow(&w, precision, k1); printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n"); - SInterval offset_1n = createInterval(1*86400*1000L, 1*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); + SInterval offset_1n = createInterval(10*86400*1000L, 10*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&offset_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + } From 7d23b176e593cdbf198b77876248e803076ae5dc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 11 May 2023 09:09:32 +0800 Subject: [PATCH 005/238] meta/table: new api metaTrimTables for vnode split --- source/dnode/vnode/src/meta/metaTable.c | 44 ++++++++++++++++++++----- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 3325f4055c..9562b1f462 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -837,22 +837,50 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi return 0; } +static void metaDropTables(SMeta *pMeta, SArray *tbUids) { + metaWLock(pMeta); + for (int i = 0; i < TARRAY_SIZE(tbUids); ++i) { + tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i); + metaDropTableByUid(pMeta, uid, NULL); + metaDebug("batch drop table:%" PRId64, uid); + } + metaULock(pMeta); +} + +int metaTrimTables(SMeta *pMeta, int64_t version) { + int32_t code = 0; + + SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); + if (tbUids == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + // code = metaFilterTableByHash(pMeta, /*ttl, */ tbUids); + if (code != 0) { + goto end; + } + if (TARRAY_SIZE(tbUids) == 0) { + goto end; + } + + metaDropTables(pMeta, tbUids); + +end: + taosArrayDestroy(tbUids); + + return code; +} + int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { int ret = metaTtlSmaller(pMeta, ttl, tbUids); if (ret != 0) { return ret; } - if (taosArrayGetSize(tbUids) == 0) { + if (TARRAY_SIZE(tbUids) == 0) { return 0; } - metaWLock(pMeta); - for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { - tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i); - metaDropTableByUid(pMeta, *uid, NULL); - metaDebug("ttl drop table:%" PRId64, *uid); - } - metaULock(pMeta); + metaDropTables(pMeta, tbUids); return 0; } From 2400104ec80fc98ac91a8c8d8253374565601cfe Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 11 May 2023 14:26:42 +0800 Subject: [PATCH 006/238] test: init add eco-system --- tests/system-test/eco-system/main.py | 37 +++ .../eco-system/manager/drop_table.py | 146 +++++++++++ .../eco-system/manager/same_column.py | 181 +++++++++++++ .../eco-system/manager/schema_change.py | 239 ++++++++++++++++++ .../eco-system/schemaless/insert.py | 147 +++++++++++ 5 files changed, 750 insertions(+) create mode 100644 tests/system-test/eco-system/main.py create mode 100644 tests/system-test/eco-system/manager/drop_table.py create mode 100644 tests/system-test/eco-system/manager/same_column.py create mode 100644 tests/system-test/eco-system/manager/schema_change.py create mode 100644 tests/system-test/eco-system/schemaless/insert.py diff --git a/tests/system-test/eco-system/main.py b/tests/system-test/eco-system/main.py new file mode 100644 index 0000000000..a1f72147a0 --- /dev/null +++ b/tests/system-test/eco-system/main.py @@ -0,0 +1,37 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + def run(self): + tdLog.info(" ------ eco-system main -------") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/eco-system/manager/drop_table.py b/tests/system-test/eco-system/manager/drop_table.py new file mode 100644 index 0000000000..d4cdf4a541 --- /dev/null +++ b/tests/system-test/eco-system/manager/drop_table.py @@ -0,0 +1,146 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(0, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + # show tables + def show_tables(self): + sql = "show tables;" + row_cnt = tdSql.query(sql) + tables = [] + for i in range(0, row_cnt): + tb_name = tdSql.queryResult[i][0] + tables.append(tb_name) + + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def drop_table(self, change_cnt): + # init + + tables = self.show_tables() + + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.drop_table(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/same_column.py b/tests/system-test/eco-system/manager/same_column.py new file mode 100644 index 0000000000..beaf4e449e --- /dev/null +++ b/tests/system-test/eco-system/manager/same_column.py @@ -0,0 +1,181 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'bc': 'bool', + 'fc': 'float', + 'dc': 'double', + 'ti': 'tinyint', + 'si': 'smallint', + 'ic': 'int', + 'bi': 'bigint', + 'uit': 'tinyint unsigned', + 'usi': 'smallint unsigned', + 'ui': 'int unsigned', + 'ubi': 'bigint unsigned', + 'bin': 'binary(32)', + 'nch': 'nchar(64)' + } + self.tag_dict = { + 'groupid': 'tinyint', + 'location': 'binary(16)', + 'tfc': 'float', + 'tdc': 'double', + 'tti': 'tinyint', + 'tsi': 'smallint', + 'tic': 'int', + 'tbi': 'bigint', + 'tuit': 'tinyint unsigned', + 'tusi': 'smallint unsigned', + 'tui': 'int unsigned', + 'tubi': 'bigint unsigned', + 'tbin': 'binary(32)', + 'tnch': 'nchar(64)' + } + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_columns(self, change_cnt): + # init + + ncol = len(self.column_dict) + ntag = len(self.tag_dict) + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_columns(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/schema_change.py b/tests/system-test/eco-system/manager/schema_change.py new file mode 100644 index 0000000000..400d2b100b --- /dev/null +++ b/tests/system-test/eco-system/manager/schema_change.py @@ -0,0 +1,239 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'col0': 'int', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'varchar(20)', + 'col13': 'nchar(20)' + } + self.tag_dict = { + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': 'varchar(20)', + 't13': 'nchar(20)', + 't14': 'timestamp' + } + + + # delete + def delete_col(self, columns, cnt, max_col): + # delte for random + for i in range(cnt): + col_cnt = len(columns) + if col_cnt == 0: + return + sel = random.randint(0, col_cnt - 1) + sql = f"alter table meters drop column `{columns[sel]}`" + try: + tdSql.execute(sql) + tdLog.info(f" drop cur col={len(columns)} max_col={max_col} {sql}") + del columns[sel] + except: + tdLog.info(f" drop column failed. {sql}") + traceback.print_exc() + + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def renames(self, tags, cnt): + col_cnt = len(tags) + if col_cnt < 10: + return + for i in range(cnt): + sel = random.randint(1, col_cnt-3) + new_name = tags[sel] + "n" + sql = f"alter table meters rename tag `{tags[sel]}` `{new_name}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + tags[sel] = new_name + except: + tdLog.info(f" rename tag failed. {sql}") + traceback.print_exc() + + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # query + def query_table(self, columns, tags): + if len(columns) < 10 : + return + if len(tags) < 10: + return + + sel_cols = random.sample(columns, random.randint(2,int(len(columns)/10))) + sel_tags = random.sample(tags, random.randint(1,int(len(tags)/10))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_schema(self, change_cnt): + # init + columns, tags = self.describe_table("meters") + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + for i in range(change_cnt): + col_cnt = len(self.column_dict) + icol = random.randint(0, col_cnt-1) + key = f"col{icol}" + col_name = key + f"_{i}_{random.randint(1,100)}" + col_type = self.column_dict[key] + sql = f'alter table meters add column `{col_name}` {col_type}' + sql_tag = f'alter table meters add tag `t_{col_name}` {col_type}' + + try: + tdSql.execute(sql) + tdLog.info(f" add cur col={len(columns)} max_col={max_col} {sql}") + columns.append(col_name) + if random.randint(1, 4) == 2: + tdSql.execute(sql_tag) + tdLog.info(f" add tag tag_cnt={len(tags)} {sql_tag}") + + except: + tdLog.info(f" add column failed. {sql}") + traceback.print_exc() + + + col_cnt = len(columns) + # delete + if col_cnt > max_col + 100: + self.delete_col(columns, random.randint(1, 30), max_col) + elif col_cnt >= max_col + 30: + self.delete_col(columns, random.randint(1, 4), max_col) + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + elif col_cnt > max_col: + self.delete_col(columns, random.randint(1, 3), max_col) + + + + if i % 50 == 0: + sql = f"flush database {self.dbname};" + tdSql.execute(sql) + tdLog.info(f" ***** {sql} *****") + + # query + if i % 70 == 0: + self.query_table(columns, tags) + + # drop and rename + if i % 10 == 0: + # update columns + columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(columns)} tags= {len(tags)}======") + + if random.randint(1,3) == 2: + self.query_table(columns, tags) + + if len(tags) > 50: + self.drop_tag(tags, random.randint(1, 30)) + + self.renames(tags, random.randint(1, 10)) + + + # sleep + #time.sleep(0.3) + + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_schema(1000000) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/schemaless/insert.py b/tests/system-test/eco-system/schemaless/insert.py new file mode 100644 index 0000000000..def70748d7 --- /dev/null +++ b/tests/system-test/eco-system/schemaless/insert.py @@ -0,0 +1,147 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback +import taos +import string +from taos import schemaless + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.conn = conn + self.schema = {} + + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + def genCol(self, col_name, isTag): + col_types = ["str","f64","f32","i8","u8","i16","u16","i32","u32","i64","u64"] + if self.schema.get(col_name) == None: + col_type = random.choice(col_types) + self.schema[col_name] = col_type + else: + col_type = self.schema[col_name] + + is_num = True + val = "" + if col_type == "str": + val = self.random_string(random.randint(1, 10)) + is_num = False + elif col_type == "f64": + val = random.randrange(-100000000000000, 1000000000000)/3*2.25678 + elif col_type == "f32": + val = random.randrange(-100000000, 1000000000)/3*1.2345 + elif col_type == "i8": + val = random.randint(-128, 127) + elif col_type == "u8": + val = random.randint(0, 256) + elif col_type == "i16": + val = random.randint(-32768, 32767) + elif col_type == "u16": + val = random.randint(0, 256*256) + elif col_type == "i32": + val = random.randint(-256*256*256*128, 256*256*256*128) + elif col_type == "u32": + val = random.randint(0, 256*256*256*256) + elif col_type == "i64": + val = random.randint(-256*256*256*256*256*256*256*128, 256*256*256*256*256*256*256*128) + elif col_type == "u64": + val = random.randint(0, 256*256*256*256*256*256*256*256) + else: + val = 100 + + if isTag: + col_val = val + elif is_num: + col_val = f'{val}{col_type}' + else: + col_val = '"' + val + '"' + + return f'{col_name}={col_val}' + + + # cols + def genCols(self, pre, max, index, isTag): + col_cnt = random.randint(1, max) + cols = [] + for i in range(col_cnt): + col_name = f'{pre}_{index}_{i}' + cols.append(self.genCol(col_name, isTag)) + + return ",".join(cols) + + + # execute sql + def insert(self,sql,i): + print("schema less insert") + try: + self.conn.schemaless_insert([sql], schemaless.SmlProtocol.LINE_PROTOCOL, schemaless.SmlPrecision.MILLI_SECONDS) + tdLog.info(f" exec ok i={i} {sql}") + except: + tdLog.info(f" exe failed. i={i} {sql}") + traceback.print_exc() + + + # change table schema + def schemaless_insert(self, change_cnt): + # init + ts = 1683194263000 + for i in range(change_cnt): + t1 = i % 1000 + index = int(i/10000) % 600 + cols = self.genCols("c", 5, index, False) + tags = f"t1={t1},t2=abc,t3=work" + sql = f'sta,{tags} {cols} {ts + i}' + self.insert(sql, i) + + # run + def run(self): + # seed + #random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + tdSql.execute(f"drop table if exists sta;") + + + + # change meters + try: + self.schemaless_insert(1000) + except: + traceback.print_exc() + + print(self.schema) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 47a1d64125b1bf4f79535e7d715c49e14c1c2b87 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 15 May 2023 11:32:08 +0800 Subject: [PATCH 007/238] enh: refactor some func names for syncLogReplProcessReply --- source/libs/sync/inc/syncPipeline.h | 7 +++---- source/libs/sync/src/syncPipeline.c | 10 +++++----- source/libs/sync/src/syncReplication.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 02790732a2..65e2cc22a0 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -79,17 +79,16 @@ static FORCE_INLINE int32_t syncLogReplGetNextRetryBackoff(SSyncLogReplMgr* pMgr SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplStart(SSyncLogReplMgr* pMgr, SSyncNode* pNode); int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); - int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId, bool* pBarrier); int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplContinue(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 8bb72de518..794d80bbfa 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -717,7 +717,7 @@ _out: return ret; } -int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SRaftId destId = pMsg->srcId; ASSERT(pMgr->restored == false); @@ -820,15 +820,15 @@ int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncApp } if (pMgr->restored) { - (void)syncLogReplProcessReplyAsNormal(pMgr, pNode, pMsg); + (void)syncLogReplContinue(pMgr, pNode, pMsg); } else { - (void)syncLogReplProcessReplyAsRecovery(pMgr, pNode, pMsg); + (void)syncLogReplRecover(pMgr, pNode, pMsg); } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplStart(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->restored) { (void)syncLogReplAttempt(pMgr, pNode); } else { @@ -931,7 +931,7 @@ int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { return 0; } -int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplContinue(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { ASSERT(pMgr->restored == true); if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 77e73402fa..015593948b 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -74,7 +74,7 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { continue; } SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - (void)syncLogReplDoOnce(pMgr, pNode); + (void)syncLogReplStart(pMgr, pNode); } return 0; } From 629ab2b85fb59eeebc3e4634de1db0c9b0b91586 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 15 May 2023 16:10:11 +0800 Subject: [PATCH 008/238] fix: extract tag equal condition --- include/libs/nodes/plannodes.h | 2 + source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 18 ++++-- source/libs/nodes/src/nodesMsgFuncs.c | 10 ++- source/libs/nodes/src/nodesUtilFuncs.c | 2 + source/libs/planner/src/planOptimizer.c | 73 ++++++++++++++++++++++ source/libs/planner/src/planPhysiCreater.c | 3 + 7 files changed, 103 insertions(+), 6 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index ad4b59714c..0e235191b1 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,6 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; + SNode* pTagEqualConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -405,6 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; + SNode* pTagEqualCondtions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index d9a4c5178f..91db527a02 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,6 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); + CLONE_NODE_FIELD(pTagEqualConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0aeb83ce08..b4e2d95e26 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1416,6 +1416,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanMergeCondition = "MergeConditions"; +static const char* jkJoinLogicPlanTagEqualConditions = "TagEqualConditions"; static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; @@ -1430,7 +1431,9 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pTagEqualConditions); + } return code; } @@ -1447,7 +1450,9 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pTagEqualConditions); + } return code; } @@ -1878,6 +1883,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition"; static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; +static const char* jkJoinPhysiPlanTagEqualConditions = "TagEqualConditions"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; @@ -1898,7 +1904,9 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pTagEqualCondtions); + } return code; } @@ -1921,7 +1929,9 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pTagEqualCondtions); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 6c6b6c0e81..45cebb4559 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2317,7 +2317,8 @@ enum { PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, PHY_SORT_MERGE_JOIN_CODE_TARGETS, - PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER + PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, + PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS }; static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2339,7 +2340,9 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pTagEqualCondtions); + } return code; } @@ -2368,6 +2371,9 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER: code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; + case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagEqualCondtions); + break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f71eef7969..d97b17e30c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1072,6 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); + nodesDestroyNode(pLogicNode->pTagEqualConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1204,6 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); + nodesDestroyNode(pPhyNode->pTagEqualCondtions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 66d85a9c89..f5ef02e207 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -740,6 +740,75 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin return code; } +static bool pushDownCondOptIsTag(SNode* pNode, SNodeList* pTableCols) { + if (QUERY_NODE_COLUMN != nodeType(pNode)) { + return false; + } + SColumnNode* pCol = (SColumnNode*)pNode; + if (COLUMN_TYPE_TAG != pCol->colType) { + return false; + } + return pushDownCondOptBelongThisTable(pNode, pTableCols); +} + +static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { + if (QUERY_NODE_OPERATOR != nodeType(pCond)) { + return false; + } + SOperatorNode* pOper = (SOperatorNode*)pCond; + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + if (pushDownCondOptIsTag(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsTag(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsTag(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsTag(pOper->pRight, pLeftCols); + } + return false; +} + +static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); + + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pTagEqualConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (pushDownCondOptIsTagEqualCond(pJoin, pCond)) { + code = nodesListMakeAppend(&pTagEqualConds, nodesCloneNode(pCond)); + } + } + + SNode* pTempTagEqCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempTagEqCond, &pTagEqualConds); + } + + if (TSDB_CODE_SUCCESS == code) { + pJoin->pTagEqualConditions = pTempTagEqCond; + return TSDB_CODE_SUCCESS; + } else { + nodesDestroyList(pTagEqualConds); + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t pushDownCondOptJoinExtractTagEqualCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { + return pushDownCondOptJoinExtractTagEqualLogicCond(pJoin); + } + + if (pushDownCondOptIsTagEqualCond(pJoin, pJoin->pOnConditions)) { + pJoin->pTagEqualConditions = nodesCloneNode(pJoin->pOnConditions); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { return TSDB_CODE_SUCCESS; @@ -774,6 +843,10 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin); } + if (TSDB_CODE_SUCCESS == code) { + code = pushDownCondOptJoinExtractTagEqualCond(pCxt, pJoin); + } + if (TSDB_CODE_SUCCESS == code) { OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e2c2e4c655..9c6c227480 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,6 +705,9 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pTagEqualConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pTagEqualConditions, &pJoin->pTagEqualCondtions); + } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); } From 789b0c7178e5ce21ea5e4231952066d9562c6023 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 16 May 2023 15:42:30 +0800 Subject: [PATCH 009/238] fix: select , from ... return error data when strlen(expr1) > TSDB_COL_NAME_LEN --- source/libs/nodes/src/nodesToSQLFuncs.c | 4 +-- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/src/parAstCreater.c | 15 +++++++++-- source/libs/parser/src/parTranslater.c | 4 ++- source/libs/planner/src/planLogicCreater.c | 1 + tests/parallel_test/cases.task | 1 + tests/script/tsim/query/bug3398.sim | 30 ++++++++++++++++++++++ 7 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 tests/script/tsim/query/bug3398.sim diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 0181da92a9..b57bba0cc9 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -120,9 +120,9 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } if (colNode->tableAlias[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->node.userAlias); } else { - *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->node.userAlias); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f71eef7969..acd028d8ae 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2054,7 +2054,7 @@ char* nodesGetNameFromColumnNode(SNode* pNode) { return "NULL"; } - return ((SColumnNode*)pNode)->colName; + return ((SColumnNode*)pNode)->node.userAlias; } int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots) { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5a47ed731d..6f866c027e 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -259,8 +259,19 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); - strncpy(pExpr->aliasName, pRawExpr->p, len); - pExpr->aliasName[len] = '\0'; + + // See TS-3398. + // Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN]. + // If aliasName is truncated, hash value of aliasName could be the same. + T_MD5_CTX ctx; + tMD5Init(&ctx); + tMD5Update(&ctx, (uint8_t*)pRawExpr->p, pRawExpr->n); + tMD5Final(&ctx); + char* p = pExpr->aliasName; + for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { + sprintf(p, "%02x", ctx.digest[i]); + p += 2; + } strncpy(pExpr->userAlias, pRawExpr->p, len); pExpr->userAlias[len] = '\0'; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f4c86d4849..9a45add98f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -827,7 +827,7 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum strcpy(pCol->node.aliasName, pCol->colName); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pCol->colName); + strcpy(pCol->node.userAlias, pExpr->userAlias); } pCol->node.resType = pExpr->resType; } @@ -1760,6 +1760,7 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName); + strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias); pVal->node.resType = ((SExprNode*)*pNode)->resType; if (NULL == pLiteral) { pVal->isNull = true; @@ -2739,6 +2740,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { } else { len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1)); + len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); strncpy(pFunc->node.userAlias, buf, TMIN(len, sizeof(pFunc->node.userAlias) - 1)); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 39783868b3..b9ea5ba0cd 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -100,6 +100,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); + strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol); diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4065ac5bee..484ac82acd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -904,6 +904,7 @@ ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/nullColSma.sim +,,y,script,./test.sh -f tsim/query/bug3398.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim ,,y,script,./test.sh -f tsim/mnode/basic1.sim diff --git a/tests/script/tsim/query/bug3398.sim b/tests/script/tsim/query/bug3398.sim new file mode 100644 index 0000000000..3ca88cf459 --- /dev/null +++ b/tests/script/tsim/query/bug3398.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database +sql create database test + +print =============== create super table and child table +sql use test + +sql CREATE STABLE st (day timestamp, c2 int) TAGS (vin binary(32)) + +sql insert into test.g using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.t using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.tg using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) + +sql select sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end), sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end) from test.t t, test.g g, test.tg tg where t.day = g.day and t.day = tg.day and t.day between '2021-05-03' and '2023-05-04' and t.vin = 'TAG1' and t.vin = g.vin and t.vin = tg.vin group by t.day; + +print $rows $data00 $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi + +if $data01 != 3.000000000 then + return -1 +endi From c6555c3b3778808e910a2044714913cd62895d61 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 16 May 2023 16:25:55 +0800 Subject: [PATCH 010/238] fix: join operator finished --- source/libs/executor/src/joinoperator.c | 369 +++++++++++++++++++----- source/libs/planner/src/planOptimizer.c | 8 + 2 files changed, 307 insertions(+), 70 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 754b5f4737..a5591b22b3 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -30,11 +30,11 @@ typedef struct SJoinRowCtx { bool rowRemains; int64_t ts; SArray* leftRowLocations; - SArray* rightRowLocations; SArray* leftCreatedBlocks; SArray* rightCreatedBlocks; int32_t leftRowIdx; int32_t rightRowIdx; + SSHashObj* buildTableTSRange; } SJoinRowCtx; typedef struct SJoinOperatorInfo { @@ -50,6 +50,17 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; + SNode* pTagEqualConditions; + + SArray* leftTagCols; + SArray* leftTagKeys; + char* leftTagKeyBuf; + int32_t leftTagKeyLen; + + SArray* rightTagCols; + SArray* rightTagKeys; + char* rightTagKeyBuf; + int32_t rightTagKeyLen; SJoinRowCtx rowCtx; } SJoinOperatorInfo; @@ -92,6 +103,147 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } +static void extractTagEqualColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, + SColumn* pLeft, SColumn* pRight) { + SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft; + SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight; + if (pLeftNode->dataBlockId == pRightNode->dataBlockId || pLeftNode->dataBlockId == pDownstreams[0]->resultDataBlockId) { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + } else { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + } +} + +static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pTagEqualNode, + SArray* leftTagEqCols, SArray* rightTagEqCols) { + SColumn left = {0}; + SColumn right = {0}; + if (nodeType(pTagEqualNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pTagEqualNode)->condType == LOGIC_COND_TYPE_AND) { + SNode* pNode = NULL; + FOREACH(pNode, ((SLogicConditionNode*)pTagEqualNode)->pParameterList) { + SOperatorNode* pOperNode = (SOperatorNode*)pNode; + extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } + return; + } + + if (nodeType(pTagEqualNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pOperNode = (SOperatorNode*)pTagEqualNode; + extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } +} + +static int32_t initTagColsGroupkeys(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { + *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); + if ((*pGroupColVals) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); + (*keyLen) += pCol->bytes; // actual data + null_flag + + SGroupKeys key = {0}; + key.bytes = pCol->bytes; + key.type = pCol->type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pCol->bytes); + if (key.pData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush((*pGroupColVals), &key); + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + (*keyLen) += nullFlagSize; + + (*keyBuf) = taosMemoryCalloc(1, (*keyLen)); + if ((*keyBuf) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static void fillGroupKeyValsFromTagCols(SArray* pCols, SArray* pGroupKeys, SSDataBlock* pBlock, int32_t rowIndex) { + SColumnDataAgg* pColAgg = NULL; + + size_t numOfGroupCols = taosArrayGetSize(pCols); + + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*) taosArrayGet(pCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + + // valid range check. todo: return error code. + if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) { + continue; + } + + if (pBlock->pBlockAgg != NULL) { + pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + + SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + pkey->isNull = true; + } else { + pkey->isNull = false; + char* val = colDataGetData(pColInfoData, rowIndex); + if (pkey->type == TSDB_DATA_TYPE_JSON) { + if (tTagIsJson(val)) { + terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; + return; + } + int32_t dataLen = getJsonValueLen(val); + memcpy(pkey->pData, val, dataLen); + } else if (IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, varDataTLen(val)); + ASSERT(varDataTLen(val) <= pkey->bytes); + } else { + memcpy(pkey->pData, val, pkey->bytes); + } + } + } +} + +static int32_t combineGroupKeysIntoBuf(void* pKey, const SArray* pGroupKeys) { + size_t numOfGroupCols = taosArrayGetSize(pGroupKeys); + + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); + if (pkey->isNull) { + isNull[i] = 1; + continue; + } + + isNull[i] = 0; + if (pkey->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(pkey->pData); + memcpy(pStart, (pkey->pData), dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pkey->type)) { + varDataCopy(pStart, pkey->pData); + pStart += varDataTLen(pkey->pData); + ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); + } else { + memcpy(pStart, pkey->pData, pkey->bytes); + pStart += pkey->bytes; + } + } + + return (int32_t)(pStart - (char*)pKey); +} + SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); @@ -153,6 +305,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } + pInfo->pTagEqualConditions = pJoinNode->pTagEqualCondtions; + pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); + extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); + initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); + initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -179,8 +338,22 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } +static void freeGroupKeyData(void* param) { + SGroupKeys* pKey = (SGroupKeys*)param; + taosMemoryFree(pKey->pData); +} + void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; + + taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->rightTagCols); + + taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->leftTagCols); + nodesDestroyNode(pJoinOperator->pCondAfterMerge); pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); @@ -300,22 +473,121 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } +static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj** ppHashObj) { + int32_t buildTableCap = MIN(taosArrayGetSize(rightRowLocations), 4096); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); + for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { + SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); + fillGroupKeyValsFromTagCols(pInfo->rightTagCols, pInfo->rightTagKeys, rightRow->pDataBlock, rightRow->pos); + int32_t keyLen = combineGroupKeysIntoBuf(pInfo->rightTagKeyBuf, pInfo->rightTagKeys); + SArray** ppRows = tSimpleHashGet(buildTable, pInfo->rightTagKeyBuf, keyLen); + if (!ppRows) { + SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); + taosArrayPush(rows, rightRow); + tSimpleHashPut(buildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); + } else { + taosArrayPush(*ppRows, rightRow); + } + } + *ppHashObj = buildTable; + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, + const SArray* leftRowLocations, int32_t leftRowIdx, + int32_t rightRowIdx, SSHashObj* rightTableHash, bool* pReachThreshold) { + *pReachThreshold = false; + uint32_t limitRowNum = pOperator->resultInfo.threshold; + SJoinOperatorInfo* pJoinInfo = pOperator->info; + size_t leftNumJoin = taosArrayGetSize(leftRowLocations); + + int32_t i,j; + + for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { + SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); + fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); + int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); + SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); + if (!ppRightRows) { + continue; + } + SArray* pRightRows = *ppRightRows; + size_t rightRowsSize = taosArrayGetSize(pRightRows); + for (j = rightRowIdx; j < rightRowsSize; ++j) { + if (*nRows >= limitRowNum) { + *pReachThreshold = true; + break; + } + + SRowLocation* rightRow = taosArrayGet(pRightRows, j); + mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, + rightRow->pos); + ++*nRows; + } + if (*pReachThreshold) { + break; + } + } + + if (*pReachThreshold) { + pJoinInfo->rowCtx.rowRemains = true; + pJoinInfo->rowCtx.leftRowIdx = i; + pJoinInfo->rowCtx.rightRowIdx = j; + } + return TSDB_CODE_SUCCESS; +} + +static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + + tSimpleHashCleanup(pBuildTable); +} + +static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, + SArray* rightCreatedBlocks, SSHashObj* rightTableHash) { + for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); + blockDataDestroy(pBlock); + } + taosArrayDestroy(rightCreatedBlocks); + for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); + blockDataDestroy(pBlock); + } + mergeJoinDestoryBuildTable(rightTableHash); + + taosArrayDestroy(leftCreatedBlocks); + taosArrayDestroy(leftRowLocations); + + pJoinInfo->rowCtx.rowRemains = false; + pJoinInfo->rowCtx.leftRowLocations = NULL; + pJoinInfo->rowCtx.leftCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightCreatedBlocks = NULL; + pJoinInfo->rowCtx.buildTableTSRange = NULL; +} + static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, int32_t* nRows) { int32_t code = TSDB_CODE_SUCCESS; SJoinOperatorInfo* pJoinInfo = pOperator->info; SArray* leftRowLocations = NULL; SArray* leftCreatedBlocks = NULL; - SArray* rightRowLocations = NULL; SArray* rightCreatedBlocks = NULL; int32_t leftRowIdx = 0; int32_t rightRowIdx = 0; - int32_t i, j; - + SSHashObj* rightTableHash = NULL; + if (pJoinInfo->rowCtx.rowRemains) { leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; - rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; + rightTableHash = pJoinInfo->rowCtx.buildTableTSRange; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; rightRowIdx = pJoinInfo->rowCtx.rightRowIdx; @@ -323,85 +595,42 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t leftRowLocations = taosArrayInit(8, sizeof(SRowLocation)); leftCreatedBlocks = taosArrayInit(8, POINTER_BYTES); - rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); + SArray* rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); rightCreatedBlocks = taosArrayInit(8, POINTER_BYTES); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 0, pJoinInfo->leftCol.slotId, pJoinInfo->pLeft, pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); + mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); + taosArrayDestroy(rightRowLocations); } size_t leftNumJoin = taosArrayGetSize(leftRowLocations); - size_t rightNumJoin = taosArrayGetSize(rightRowLocations); - uint32_t maxRowNum = *nRows + (leftNumJoin - leftRowIdx - 1) * rightNumJoin + rightNumJoin - rightRowIdx; - uint32_t limitRowNum = maxRowNum; - if (maxRowNum > pOperator->resultInfo.threshold) { - limitRowNum = pOperator->resultInfo.threshold; - if (!pJoinInfo->rowCtx.rowRemains) { + code = blockDataEnsureCapacity(pRes, pOperator->resultInfo.threshold); + if (code != TSDB_CODE_SUCCESS) { + qError("%s can not ensure block capacity for join. left: %zu", GET_TASKID(pOperator->pTaskInfo), + leftNumJoin); + } + + bool reachThreshold = false; + + if (code == TSDB_CODE_SUCCESS) { + mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, + rightRowIdx, rightTableHash, &reachThreshold); + } + + if (!reachThreshold) { + mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, + rightTableHash); + + } else { pJoinInfo->rowCtx.rowRemains = true; pJoinInfo->rowCtx.ts = timestamp; pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; - pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - } - } - - code = blockDataEnsureCapacity(pRes, limitRowNum); - if (code != TSDB_CODE_SUCCESS) { - qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo), - leftNumJoin, rightNumJoin); - } - - - if (code == TSDB_CODE_SUCCESS) { - bool done = false; - for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { - for (j = rightRowIdx; j < rightNumJoin; ++j) { - if (*nRows >= limitRowNum) { - done = true; - break; - } - - SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); - SRowLocation* rightRow = taosArrayGet(rightRowLocations, j); - mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, - rightRow->pos); - ++*nRows; - } - if (done) { - break; - } - } - - if (maxRowNum > pOperator->resultInfo.threshold) { - pJoinInfo->rowCtx.leftRowIdx = i; - pJoinInfo->rowCtx.rightRowIdx = j; - } - } - - if (maxRowNum <= pOperator->resultInfo.threshold) { - for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(rightCreatedBlocks); - taosArrayDestroy(rightRowLocations); - for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(leftCreatedBlocks); - taosArrayDestroy(leftRowLocations); - - if (pJoinInfo->rowCtx.rowRemains) { - pJoinInfo->rowCtx.rowRemains = false; - pJoinInfo->rowCtx.leftRowLocations = NULL; - pJoinInfo->rowCtx.rightRowLocations = NULL; - pJoinInfo->rowCtx.leftCreatedBlocks = NULL; - pJoinInfo->rowCtx.rightCreatedBlocks = NULL; - } + pJoinInfo->rowCtx.buildTableTSRange = rightTableHash; } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index f5ef02e207..8c111133fc 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -759,6 +759,14 @@ static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (OP_TYPE_EQUAL != pOper->opType) { return false; } + if (QUERY_NODE_COLUMN != nodeType(pOper->pLeft) || QUERY_NODE_COLUMN != nodeType(pOper->pRight)) { + return false; + } + SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft); + SColumnNode* pRight = (SColumnNode*)(pOper->pRight); + if (pLeft->node.resType.type != pRight->node.resType.type) { + return false; + } SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; if (pushDownCondOptIsTag(pOper->pLeft, pLeftCols)) { From a51de5cc9384b22d09ec27f9607f46361e548975 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 16 May 2023 16:31:58 +0800 Subject: [PATCH 011/238] fix: add restriction --- source/libs/planner/src/planOptimizer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8c111133fc..d76c25fc55 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -764,7 +764,8 @@ static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { } SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft); SColumnNode* pRight = (SColumnNode*)(pOper->pRight); - if (pLeft->node.resType.type != pRight->node.resType.type) { + //TODO: add cast to operator and remove this restriction of optimization + if (pLeft->node.resType.type != pRight->node.resType.type || pLeft->node.resType.bytes != pRight->node.resType.bytes) { return false; } SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; From 568725a245d8e1a2f80dc056c85955bdd8a382cd Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 17 May 2023 08:33:47 +0800 Subject: [PATCH 012/238] fix: using right rows to build hash table when the tag equal cond is not null and same ts row row number is greater than 16 --- source/libs/executor/src/joinoperator.c | 74 ++++++++++++++++--------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index a5591b22b3..54ddab9e1e 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -34,7 +34,9 @@ typedef struct SJoinRowCtx { SArray* rightCreatedBlocks; int32_t leftRowIdx; int32_t rightRowIdx; + SSHashObj* buildTableTSRange; + SArray* rightRowLocations; } SJoinRowCtx; typedef struct SJoinOperatorInfo { @@ -306,12 +308,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t } pInfo->pTagEqualConditions = pJoinNode->pTagEqualCondtions; - pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); - pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); - extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); - initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); - initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); - + if (pInfo->pTagEqualConditions != NULL) { + pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); + extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); + initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); + initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -345,15 +348,15 @@ static void freeGroupKeyData(void* param) { void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; + if (pJoinOperator->pTagEqualConditions != NULL) { + taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->rightTagCols); - taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); - taosArrayDestroy(pJoinOperator->rightTagCols); - - taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); - taosArrayDestroy(pJoinOperator->leftTagCols); - + taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->leftTagCols); + } nodesDestroyNode(pJoinOperator->pCondAfterMerge); pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); @@ -496,7 +499,7 @@ static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* right static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, const SArray* leftRowLocations, int32_t leftRowIdx, - int32_t rightRowIdx, SSHashObj* rightTableHash, bool* pReachThreshold) { + int32_t rightRowIdx, SSHashObj* rightTableHash, SArray* rightRowLocations, bool* pReachThreshold) { *pReachThreshold = false; uint32_t limitRowNum = pOperator->resultInfo.threshold; SJoinOperatorInfo* pJoinInfo = pOperator->info; @@ -506,13 +509,18 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); - fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); - int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); - SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); - if (!ppRightRows) { - continue; + SArray* pRightRows = NULL; + if (rightTableHash != NULL) { + fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); + int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); + SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); + if (!ppRightRows) { + continue; + } + pRightRows = *ppRightRows; + } else { + pRightRows = rightRowLocations; } - SArray* pRightRows = *ppRightRows; size_t rightRowsSize = taosArrayGetSize(pRightRows); for (j = rightRowIdx; j < rightRowsSize; ++j) { if (*nRows >= limitRowNum) { @@ -551,7 +559,7 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { } static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, - SArray* rightCreatedBlocks, SSHashObj* rightTableHash) { + SArray* rightCreatedBlocks, SSHashObj* rightTableHash, SArray* rightRowLocations) { for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); blockDataDestroy(pBlock); @@ -561,7 +569,12 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); blockDataDestroy(pBlock); } - mergeJoinDestoryBuildTable(rightTableHash); + if (rightRowLocations != NULL) { + taosArrayDestroy(rightRowLocations); + } + if (rightTableHash != NULL) { + mergeJoinDestoryBuildTable(rightTableHash); + } taosArrayDestroy(leftCreatedBlocks); taosArrayDestroy(leftRowLocations); @@ -571,6 +584,7 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef pJoinInfo->rowCtx.leftCreatedBlocks = NULL; pJoinInfo->rowCtx.rightCreatedBlocks = NULL; pJoinInfo->rowCtx.buildTableTSRange = NULL; + pJoinInfo->rowCtx.rightRowLocations = NULL; } static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, @@ -578,6 +592,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t int32_t code = TSDB_CODE_SUCCESS; SJoinOperatorInfo* pJoinInfo = pOperator->info; SArray* leftRowLocations = NULL; + SArray* rightRowLocations = NULL; SArray* leftCreatedBlocks = NULL; SArray* rightCreatedBlocks = NULL; int32_t leftRowIdx = 0; @@ -588,6 +603,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; rightTableHash = pJoinInfo->rowCtx.buildTableTSRange; + rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; rightRowIdx = pJoinInfo->rowCtx.rightRowIdx; @@ -602,8 +618,11 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); - mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); - taosArrayDestroy(rightRowLocations); + if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); + taosArrayDestroy(rightRowLocations); + rightRowLocations = NULL; + } } size_t leftNumJoin = taosArrayGetSize(leftRowLocations); @@ -617,12 +636,12 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t if (code == TSDB_CODE_SUCCESS) { mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, - rightRowIdx, rightTableHash, &reachThreshold); + rightRowIdx, rightTableHash, rightRowLocations, &reachThreshold); } if (!reachThreshold) { mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, - rightTableHash); + rightTableHash, rightRowLocations); } else { pJoinInfo->rowCtx.rowRemains = true; @@ -631,6 +650,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; pJoinInfo->rowCtx.buildTableTSRange = rightTableHash; + pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; } From 03e77288e8ca2b3ea9a7385c21f181593fa899a6 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 17 May 2023 10:07:19 +0800 Subject: [PATCH 013/238] fix: add on conditions null check and rightRowLocations no longer hides the outer scope rightRowLocations --- source/libs/executor/src/joinoperator.c | 4 ++-- source/libs/planner/src/planOptimizer.c | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 54ddab9e1e..07b8a4db11 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -477,7 +477,7 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator } static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj** ppHashObj) { - int32_t buildTableCap = MIN(taosArrayGetSize(rightRowLocations), 4096); + int32_t buildTableCap = TMIN(taosArrayGetSize(rightRowLocations), 256); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { @@ -611,7 +611,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t leftRowLocations = taosArrayInit(8, sizeof(SRowLocation)); leftCreatedBlocks = taosArrayInit(8, POINTER_BYTES); - SArray* rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); + rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); rightCreatedBlocks = taosArrayInit(8, POINTER_BYTES); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 0, pJoinInfo->leftCol.slotId, pJoinInfo->pLeft, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d76c25fc55..5be67389c8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -806,6 +806,10 @@ static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin } static int32_t pushDownCondOptJoinExtractTagEqualCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (NULL == pJoin->pOnConditions) { + pJoin->pTagEqualConditions = NULL; + return TSDB_CODE_SUCCESS; + } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { return pushDownCondOptJoinExtractTagEqualLogicCond(pJoin); From 28b098c824a7e008f6317d85b91ea29d15159d0a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 May 2023 10:47:11 +0800 Subject: [PATCH 014/238] fix: union clause error message --- source/util/src/terror.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6f726c7eff..712c413e7c 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -372,7 +372,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many groups/time window in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") From 1ded7a052bfc5fd9e823236d35800c5dc1fd3996 Mon Sep 17 00:00:00 2001 From: Yu Sun Date: Wed, 17 May 2023 07:22:15 +0000 Subject: [PATCH 015/238] docs(connector-rust): update documents of connector for rust The connector for rust has been updated to use `deadpool` as default connection pool instead of `r2d2`, and the `r2d2` was only kept for blocking mode. --- docs/zh/08-connector/26-rust.mdx | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index eeb93a6558..5cf37f7946 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -64,6 +64,13 @@ taos = "*" taos = { version = "*", default-features = false, features = ["ws"] } ``` +当仅启用 `ws` 特性时，可同时指定 `r2d2` 使得在同步（blocking/sync）模式下使用 [r2d2] 作为连接池： + +```toml +[dependencies] +taos = { version = "*", default-features = false, features = ["r2d2", "ws"] } +``` + @@ -252,26 +259,24 @@ let conn: Taos = cfg.build(); ### 连接池 -在复杂应用中，建议启用连接池。[taos] 的连接池使用 [r2d2] 实现。 +在复杂应用中，建议启用连接池。[taos] 的连接池默认（异步模式）使用 [deadpool] 实现。 如下，可以生成一个默认参数的连接池。 ```rust -let pool = TaosBuilder::from_dsn(dsn)?.pool()?; +let pool: Pool = TaosBuilder::from_dsn("taos:///") + .unwrap() + .pool() + .unwrap(); ``` 同样可以使用连接池的构造器，对连接池参数进行设置： ```rust -let dsn = "taos://localhost:6030"; - -let opts = PoolBuilder::new() - .max_size(5000) // max connections - .max_lifetime(Some(Duration::from_secs(60 * 60))) // lifetime of each connection - .min_idle(Some(1000)) // minimal idle connections - .connection_timeout(Duration::from_secs(2)); - -let pool = TaosBuilder::from_dsn(dsn)?.with_pool_builder(opts)?; +let pool: Pool = Pool::builder(Manager::from_dsn(self.dsn.clone()).unwrap().0) + .max_size(88) // 最大连接数 + .build() + .unwrap(); ``` 在应用代码中，使用 `pool.get()?` 来获取一个连接对象 [Taos]。 @@ -511,6 +516,7 @@ consumer.unsubscribe().await; 其他相关结构体 API 使用说明请移步 Rust 文档托管网页：。 [taos]: https://github.com/taosdata/rust-connector-taos +[deadpool]: https://crates.io/crates/deadpool [r2d2]: https://crates.io/crates/r2d2 [TaosBuilder]: https://docs.rs/taos/latest/taos/struct.TaosBuilder.html [TaosCfg]: https://docs.rs/taos/latest/taos/struct.TaosCfg.html From ab7cb995a2eb87fd9c32168d1874cf99a3f07012 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 May 2023 15:31:53 +0800 Subject: [PATCH 016/238] fix: join push down condition issue --- source/libs/nodes/src/nodesEqualFuncs.c | 4 +--- source/libs/parser/test/parSelectTest.cpp | 2 ++ source/libs/planner/src/planPhysiCreater.c | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 156744ef1d..477d27d271 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -82,9 +82,7 @@ static bool columnNodeEqual(const SColumnNode* a, const SColumnNode* b) { COMPARE_STRING_FIELD(dbName); COMPARE_STRING_FIELD(tableName); COMPARE_STRING_FIELD(colName); - if (0 == a->tableId) { - COMPARE_STRING_FIELD(tableAlias); - } + COMPARE_STRING_FIELD(tableAlias); return true; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index ec6c69ea8d..c60257cc22 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -459,6 +459,8 @@ TEST_F(ParserSelectTest, joinSemanticCheck) { run("SELECT * FROM (SELECT tag1, SUM(c1) s FROM st1 GROUP BY tag1) t1, st1 t2 where t1.tag1 = t2.tag1", TSDB_CODE_PAR_NOT_SUPPORT_JOIN); + + run("SELECT count(*) FROM t1 a join t1 b on a.ts=b.ts where a.ts=b.ts"); } } // namespace ParserTest diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e2c2e4c655..651be08aec 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1643,6 +1643,9 @@ static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pChildren, (SNode*)pChild); } + if (TSDB_CODE_SUCCESS != code) { + break; + } } if (TSDB_CODE_SUCCESS == code) { From 73b3b3fc377be4480112e4f61f32f1959e79ac24 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 May 2023 15:48:12 +0800 Subject: [PATCH 017/238] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6724e2d303..eb4f04c914 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -291,13 +291,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - if (offset.val.type == TMQ_OFFSET__LOG) { - STqHandle* pHandle = taosHashGet(pTq->pHandle, offset.subKey, strlen(offset.subKey)); - if (pHandle && (walRefVer(pHandle->pRef, offset.val.version) < 0)) { - return -1; - } - } - return 0; } From 4bb25c280f0452c5b078580c1bba8b9b09a94b91 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 15:54:36 +0800 Subject: [PATCH 018/238] fix:add log --- source/dnode/vnode/src/tq/tq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9730ad2877..ff715f1828 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -359,7 +359,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } while (tqIsHandleExec(pHandle)) { - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", consumerId, vgId, req.subKey); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosMsleep(5); } @@ -372,6 +372,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } tqSetHandleExec(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); // 3. update the epoch value @@ -389,6 +390,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); tqSetHandleIdle(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; } @@ -555,7 +557,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg goto end; } else { while (tqIsHandleExec(pHandle)) { - tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); + tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); taosMsleep(5); } From 1d9dd153eacdb10e5e1d03d896ca104f1bcaf30b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 18:57:45 +0800 Subject: [PATCH 019/238] fix:error in pHandle lock --- source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/tq/tq.c | 64 +++++++++++++++------------- source/dnode/vnode/src/tq/tqMeta.c | 2 - source/dnode/vnode/src/tq/tqOffset.c | 2 - source/dnode/vnode/src/tq/tqRead.c | 2 - 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ef36b8429a..f0008f04cb 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -93,6 +93,7 @@ typedef struct { typedef enum tq_handle_status{ TMQ_HANDLE_STATUS_IDLE = 0, TMQ_HANDLE_STATUS_EXEC = 1, + TMQ_HANDLE_STATUS_DELETE = 2, }tq_handle_status; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ff715f1828..b1fca3faa8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -25,6 +25,7 @@ static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_EXEC;} static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_IDLE;} +static FORCE_INLINE void tqSetHandleDelete(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_DELETE;} int32_t tqInit() { int8_t old; @@ -297,13 +298,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } if (offset.val.type == TMQ_OFFSET__LOG) { - taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, offset.subKey, strlen(offset.subKey)); if (pHandle && (walSetRefVer(pHandle->pRef, offset.val.version) < 0)) { - taosWUnLockLatch(&pTq->lock); return -1; } - taosWUnLockLatch(&pTq->lock); } return 0; @@ -337,6 +335,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SMqPollReq req = {0}; + int code = 0; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); terrno = TSDB_CODE_INVALID_MSG; @@ -347,20 +346,29 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t reqEpoch = req.epoch; STqOffsetVal reqOffset = req.reqOffset; int32_t vgId = TD_VID(pTq->pVnode); + STqHandle* pHandle = NULL; - taosWLockLatch(&pTq->lock); - // 1. find handle - STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - if (pHandle == NULL) { - tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); - terrno = TSDB_CODE_INVALID_MSG; + while (1) { + taosWLockLatch(&pTq->lock); + // 1. find handle + pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle == NULL) { + tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; + taosWUnLockLatch(&pTq->lock); + return -1; + } + + bool exec = tqIsHandleExec(pHandle); + if(!exec) { + tqSetHandleExec(pHandle); + taosWUnLockLatch(&pTq->lock); + break; + } taosWUnLockLatch(&pTq->lock); - return -1; - } - while (tqIsHandleExec(pHandle)) { tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); - taosMsleep(5); + taosMsleep(10); } // 2. check re-balance status @@ -368,12 +376,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - taosWUnLockLatch(&pTq->lock); - return -1; + code = -1; + goto end; } - tqSetHandleExec(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); - taosWUnLockLatch(&pTq->lock); // 3. update the epoch value int32_t savedEpoch = pHandle->epoch; @@ -388,7 +395,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); - int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); + code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); + +end: tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; @@ -405,8 +414,8 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { while (tqIsHandleExec(pHandle)) { - tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); - taosMsleep(5); + tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); + taosMsleep(10); } if (pHandle->pRef) { walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); @@ -473,7 +482,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); - taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -556,20 +564,14 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { - while (tqIsHandleExec(pHandle)) { - tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); - taosMsleep(5); - } - if (pHandle->consumerId == req.newConsumerId) { // do nothing tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); - atomic_add_fetch_32(&pHandle->epoch, 1); - +// atomic_add_fetch_32(&pHandle->epoch, 1); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); - atomic_store_32(&pHandle->epoch, 0); +// atomic_store_32(&pHandle->epoch, 0); } // kill executing task qTaskInfo_t pTaskInfo = pHandle->execHandle.task; @@ -580,13 +582,15 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg qStreamCloseTsdbReader(pTaskInfo); } // remove if it has been register in the push manager, and return one empty block to consumer + taosWLockLatch(&pTq->lock); tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } end: - taosWUnLockLatch(&pTq->lock); taosMemoryFree(req.qmsg); return ret; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 5654147b6d..b3d21887c9 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -352,9 +352,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); - taosWLockLatch(&pTq->lock); taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); - taosWUnLockLatch(&pTq->lock); } end: diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 377a5d1887..0a9905b544 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -78,7 +78,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // todo remove this if (offset.val.type == TMQ_OFFSET__LOG) { - taosWLockLatch(&pStore->pTq->lock); STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); if (pHandle) { if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) { @@ -86,7 +85,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // offset.val.version); } } - taosWUnLockLatch(&pStore->pTq->lock); } taosMemoryFree(pMemBuf); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 09574a6b62..a26a749af3 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1035,7 +1035,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t vgId = TD_VID(pTq->pVnode); // update the table list for each consumer handle - taosWLockLatch(&pTq->lock); while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); if (pIter == NULL) { @@ -1092,7 +1091,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } } - taosWUnLockLatch(&pTq->lock); // update the table list handle for each stream scanner/wal reader taosWLockLatch(&pTq->pStreamMeta->lock); From 405cdfa7bc5637be46072149303bed2a03e5b20c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 19:02:16 +0800 Subject: [PATCH 020/238] fix:error in pHandle lock --- source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/tq/tq.c | 1 - 2 files changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f0008f04cb..ef36b8429a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -93,7 +93,6 @@ typedef struct { typedef enum tq_handle_status{ TMQ_HANDLE_STATUS_IDLE = 0, TMQ_HANDLE_STATUS_EXEC = 1, - TMQ_HANDLE_STATUS_DELETE = 2, }tq_handle_status; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b1fca3faa8..d8ebf72549 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -25,7 +25,6 @@ static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_EXEC;} static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_IDLE;} -static FORCE_INLINE void tqSetHandleDelete(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_DELETE;} int32_t tqInit() { int8_t old; From cf2c5f190c865003cc7c8132a158840c1e07d8b9 Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Wed, 17 May 2023 19:26:34 +0800 Subject: [PATCH 021/238] Update cases.task modify non asan mode --- tests/parallel_test/cases.task | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 5a6ec7a825..8d151f4756 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -78,7 +78,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +,,n,system-test,python3 ./test.py -f 7-tmq/tmqConsumerGroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAlterSchema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 From 4d5bd2b6dee5321d3aa4695c5553f1866cf4a37e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 May 2023 19:40:21 +0800 Subject: [PATCH 022/238] fix(stream): fix memory leak for stream processing. --- source/libs/stream/src/stream.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 7047461ca8..f4ee0b98b7 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -297,6 +297,7 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); streamDataSubmitDestroy(pSubmitBlock); + taosFreeQitem(pSubmitBlock); return -1; } From 54f48601394cf1dff9f8933d57a3887aaa8ec0d4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 19:44:56 +0800 Subject: [PATCH 023/238] fix:rollback [TD-23972] push subscribe msg to vnode even though consumer not change --- source/dnode/mnode/impl/src/mndSubscribe.c | 21 +++------------ source/dnode/vnode/src/tq/tq.c | 30 ++++++++++------------ 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e62102fa77..2ae4ede25a 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -133,10 +133,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg) { -// if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { -// terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; -// return -1; -// } + if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { + terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; + return -1; + } void *buf; int32_t tlen; @@ -269,18 +269,6 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { } } -static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ - for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ - SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); - SMqRebOutputVg outputVg = { - .oldConsumerId = pConsumerEp->consumerId, - .newConsumerId = pConsumerEp->consumerId, - .pVgEp = pVgEp, - }; - taosArrayPush(pOutput->rebVgs, &outputVg); - } -} - static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, int32_t imbConsumerNum) { const char *pSubKey = pOutput->pSub->key; @@ -330,7 +318,6 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas } } } - putNoTransferToOutput(pOutput, pConsumerEp); } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d8ebf72549..e074a2d8a6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -564,29 +564,27 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg goto end; } else { if (pHandle->consumerId == req.newConsumerId) { // do nothing - tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); + tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs, should not reach here", req.vgId, req.newConsumerId); // atomic_add_fetch_32(&pHandle->epoch, 1); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); // atomic_store_32(&pHandle->epoch, 0); + // kill executing task + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + } + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } + // remove if it has been register in the push manager, and return one empty block to consumer + taosWLockLatch(&pTq->lock); + tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } - // remove if it has been register in the push manager, and return one empty block to consumer - taosWLockLatch(&pTq->lock); - tqUnregisterPushHandle(pTq, pHandle); - taosWUnLockLatch(&pTq->lock); - - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); - goto end; } end: From 8994081592b788630e784fa3d7894ba9b9cffeff Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 May 2023 10:25:10 +0800 Subject: [PATCH 024/238] fix: memory size overflow issue --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fa46715c22..5953074825 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -135,7 +135,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in size_t keyLen = 0; int32_t iter = 0; - int32_t bufLen = 0, offset = 0; + int64_t bufLen = 0, offset = 0; // todo move away and record this during create window while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { From ee16b46112e6a83eb1434b9ab90cc104c8e89bcb Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 10:46:19 +0800 Subject: [PATCH 025/238] fix: remove group keys from join operator --- source/libs/executor/src/joinoperator.c | 94 +++++-------------------- 1 file changed, 18 insertions(+), 76 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 07b8a4db11..68f6951c26 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -55,12 +55,10 @@ typedef struct SJoinOperatorInfo { SNode* pTagEqualConditions; SArray* leftTagCols; - SArray* leftTagKeys; char* leftTagKeyBuf; int32_t leftTagKeyLen; SArray* rightTagCols; - SArray* rightTagKeys; char* rightTagKeyBuf; int32_t rightTagKeyLen; @@ -141,27 +139,11 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD } } -static int32_t initTagColsGroupkeys(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { - *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); - if ((*pGroupColVals) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - +static int32_t initTagColskeyBuf(int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); (*keyLen) += pCol->bytes; // actual data + null_flag - - SGroupKeys key = {0}; - key.bytes = pCol->bytes; - key.type = pCol->type; - key.isNull = false; - key.pData = taosMemoryCalloc(1, pCol->bytes); - if (key.pData == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - taosArrayPush((*pGroupColVals), &key); } int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; @@ -175,10 +157,11 @@ static int32_t initTagColsGroupkeys(SArray** pGroupColVals, int32_t* keyLen, cha return TSDB_CODE_SUCCESS; } -static void fillGroupKeyValsFromTagCols(SArray* pCols, SArray* pGroupKeys, SSDataBlock* pBlock, int32_t rowIndex) { +static int32_t fillKeyBufFromTagCols(SArray* pCols, SSDataBlock* pBlock, int32_t rowIndex, void* pKey) { SColumnDataAgg* pColAgg = NULL; - size_t numOfGroupCols = taosArrayGetSize(pCols); + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = (SColumn*) taosArrayGet(pCols, i); @@ -193,56 +176,24 @@ static void fillGroupKeyValsFromTagCols(SArray* pCols, SArray* pGroupKeys, SSDat pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } - SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { - pkey->isNull = true; + isNull[i] = 1; } else { - pkey->isNull = false; + isNull[i] = 0; char* val = colDataGetData(pColInfoData, rowIndex); - if (pkey->type == TSDB_DATA_TYPE_JSON) { - if (tTagIsJson(val)) { - terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; - return; - } + if (pCol->type == TSDB_DATA_TYPE_JSON) { int32_t dataLen = getJsonValueLen(val); - memcpy(pkey->pData, val, dataLen); - } else if (IS_VAR_DATA_TYPE(pkey->type)) { - memcpy(pkey->pData, val, varDataTLen(val)); - ASSERT(varDataTLen(val) <= pkey->bytes); + memcpy(pStart, val, dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + varDataCopy(pStart, val); + pStart += varDataTLen(val); } else { - memcpy(pkey->pData, val, pkey->bytes); + memcpy(pStart, val, pCol->bytes); + pStart += pCol->bytes; } } } -} - -static int32_t combineGroupKeysIntoBuf(void* pKey, const SArray* pGroupKeys) { - size_t numOfGroupCols = taosArrayGetSize(pGroupKeys); - - char* isNull = (char*)pKey; - char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; - for (int32_t i = 0; i < numOfGroupCols; ++i) { - SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); - if (pkey->isNull) { - isNull[i] = 1; - continue; - } - - isNull[i] = 0; - if (pkey->type == TSDB_DATA_TYPE_JSON) { - int32_t dataLen = getJsonValueLen(pkey->pData); - memcpy(pStart, (pkey->pData), dataLen); - pStart += dataLen; - } else if (IS_VAR_DATA_TYPE(pkey->type)) { - varDataCopy(pStart, pkey->pData); - pStart += varDataTLen(pkey->pData); - ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); - } else { - memcpy(pStart, pkey->pData, pkey->bytes); - pStart += pkey->bytes; - } - } - return (int32_t)(pStart - (char*)pKey); } @@ -312,8 +263,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); - initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); - initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + initTagColskeyBuf(&pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); + initTagColskeyBuf(&pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); @@ -341,20 +292,13 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } -static void freeGroupKeyData(void* param) { - SGroupKeys* pKey = (SGroupKeys*)param; - taosMemoryFree(pKey->pData); -} - void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; if (pJoinOperator->pTagEqualConditions != NULL) { taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); taosArrayDestroy(pJoinOperator->rightTagCols); taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); taosArrayDestroy(pJoinOperator->leftTagCols); } nodesDestroyNode(pJoinOperator->pCondAfterMerge); @@ -482,8 +426,7 @@ static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* right SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); - fillGroupKeyValsFromTagCols(pInfo->rightTagCols, pInfo->rightTagKeys, rightRow->pDataBlock, rightRow->pos); - int32_t keyLen = combineGroupKeysIntoBuf(pInfo->rightTagKeyBuf, pInfo->rightTagKeys); + int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); SArray** ppRows = tSimpleHashGet(buildTable, pInfo->rightTagKeyBuf, keyLen); if (!ppRows) { SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); @@ -511,8 +454,7 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SArray* pRightRows = NULL; if (rightTableHash != NULL) { - fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); - int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); + int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftTagCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftTagKeyBuf); SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); if (!ppRightRows) { continue; From 14d4a81e750303da755b121da8c2c36dfac93dff Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 11:15:04 +0800 Subject: [PATCH 026/238] fix: build hash table is created during creating join operator --- source/libs/executor/src/joinoperator.c | 62 +++++++++++++------------ 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 68f6951c26..c90c18e6f5 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -35,7 +35,7 @@ typedef struct SJoinRowCtx { int32_t leftRowIdx; int32_t rightRowIdx; - SSHashObj* buildTableTSRange; + bool rightUseBuildTable; SArray* rightRowLocations; } SJoinRowCtx; @@ -62,6 +62,7 @@ typedef struct SJoinOperatorInfo { char* rightTagKeyBuf; int32_t rightTagKeyLen; + SSHashObj* rightBuildTable; SJoinRowCtx rowCtx; } SJoinOperatorInfo; @@ -265,6 +266,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); initTagColskeyBuf(&pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); initTagColskeyBuf(&pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->rightBuildTable = tSimpleHashInit(256, hashFn); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); @@ -292,9 +295,22 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } +static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + + tSimpleHashCleanup(pBuildTable); +} + void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; if (pJoinOperator->pTagEqualConditions != NULL) { + mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); taosArrayDestroy(pJoinOperator->rightTagCols); @@ -420,10 +436,7 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } -static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj** ppHashObj) { - int32_t buildTableCap = TMIN(taosArrayGetSize(rightRowLocations), 256); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); +static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj* buildTable) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); @@ -436,13 +449,12 @@ static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* right taosArrayPush(*ppRows, rightRow); } } - *ppHashObj = buildTable; return TSDB_CODE_SUCCESS; } static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, const SArray* leftRowLocations, int32_t leftRowIdx, - int32_t rightRowIdx, SSHashObj* rightTableHash, SArray* rightRowLocations, bool* pReachThreshold) { + int32_t rightRowIdx, bool useBuildTableTSRange, SArray* rightRowLocations, bool* pReachThreshold) { *pReachThreshold = false; uint32_t limitRowNum = pOperator->resultInfo.threshold; SJoinOperatorInfo* pJoinInfo = pOperator->info; @@ -453,9 +465,9 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SArray* pRightRows = NULL; - if (rightTableHash != NULL) { + if (useBuildTableTSRange) { int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftTagCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftTagKeyBuf); - SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); + SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftTagKeyBuf, keyLen); if (!ppRightRows) { continue; } @@ -488,20 +500,8 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* return TSDB_CODE_SUCCESS; } -static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { - void* p = NULL; - int32_t iter = 0; - - while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { - SArray* rows = (*(SArray**)p); - taosArrayDestroy(rows); - } - - tSimpleHashCleanup(pBuildTable); -} - static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, - SArray* rightCreatedBlocks, SSHashObj* rightTableHash, SArray* rightRowLocations) { + SArray* rightCreatedBlocks, bool rightUseBuildTable, SArray* rightRowLocations) { for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); blockDataDestroy(pBlock); @@ -514,8 +514,8 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef if (rightRowLocations != NULL) { taosArrayDestroy(rightRowLocations); } - if (rightTableHash != NULL) { - mergeJoinDestoryBuildTable(rightTableHash); + if (rightUseBuildTable) { + tSimpleHashClear(pJoinInfo->rightBuildTable); } taosArrayDestroy(leftCreatedBlocks); @@ -525,7 +525,7 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef pJoinInfo->rowCtx.leftRowLocations = NULL; pJoinInfo->rowCtx.leftCreatedBlocks = NULL; pJoinInfo->rowCtx.rightCreatedBlocks = NULL; - pJoinInfo->rowCtx.buildTableTSRange = NULL; + pJoinInfo->rowCtx.rightUseBuildTable = false; pJoinInfo->rowCtx.rightRowLocations = NULL; } @@ -540,11 +540,12 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t int32_t leftRowIdx = 0; int32_t rightRowIdx = 0; SSHashObj* rightTableHash = NULL; + bool rightUseBuildTable = false; if (pJoinInfo->rowCtx.rowRemains) { leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; - rightTableHash = pJoinInfo->rowCtx.buildTableTSRange; + rightUseBuildTable = pJoinInfo->rowCtx.rightUseBuildTable; rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; @@ -561,7 +562,8 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { - mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); + mergeJoinFillBuildTable(pJoinInfo, rightRowLocations, pJoinInfo->rightBuildTable); + rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); rightRowLocations = NULL; } @@ -578,12 +580,12 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t if (code == TSDB_CODE_SUCCESS) { mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, - rightRowIdx, rightTableHash, rightRowLocations, &reachThreshold); + rightRowIdx, rightUseBuildTable, rightRowLocations, &reachThreshold); } if (!reachThreshold) { mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, - rightTableHash, rightRowLocations); + rightUseBuildTable, rightRowLocations); } else { pJoinInfo->rowCtx.rowRemains = true; @@ -591,7 +593,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - pJoinInfo->rowCtx.buildTableTSRange = rightTableHash; + pJoinInfo->rowCtx.rightUseBuildTable = true; pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; From d7f8757844da2a8db1d9a756d2d9884a3b57cb91 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 11:23:50 +0800 Subject: [PATCH 027/238] fix: memory leak --- source/libs/executor/src/joinoperator.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index c90c18e6f5..0add034673 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -515,6 +515,12 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef taosArrayDestroy(rightRowLocations); } if (rightUseBuildTable) { + void* p = NULL; + int32_t iter = 0; + while ((p = tSimpleHashIterate(pJoinInfo->rightBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } tSimpleHashClear(pJoinInfo->rightBuildTable); } From 7102c107ebd4884541bfa47d428b73020b5a7d43 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 13:19:52 +0800 Subject: [PATCH 028/238] fix: remove parameter of filling build table --- source/libs/executor/src/joinoperator.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 0add034673..23bd7c5700 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -436,15 +436,16 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } -static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj* buildTable) { +static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { + tSimpleHashClear(pInfo->rightBuildTable); SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); - SArray** ppRows = tSimpleHashGet(buildTable, pInfo->rightTagKeyBuf, keyLen); + SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen); if (!ppRows) { SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); taosArrayPush(rows, rightRow); - tSimpleHashPut(buildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); + tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); } else { taosArrayPush(*ppRows, rightRow); } @@ -568,7 +569,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { - mergeJoinFillBuildTable(pJoinInfo, rightRowLocations, pJoinInfo->rightBuildTable); + mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); rightRowLocations = NULL; From 51cb79fc1d93eec01fba74a513c7e7993c3c8e98 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 13:28:46 +0800 Subject: [PATCH 029/238] fix: set rightUseBuildTable correctly --- source/libs/executor/src/joinoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 23bd7c5700..700a59ccd8 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -600,7 +600,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - pJoinInfo->rowCtx.rightUseBuildTable = true; + pJoinInfo->rowCtx.rightUseBuildTable = rightUseBuildTable; pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; From 72e63195920db702e3b1165949d1f13e56d84c53 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 May 2023 14:41:32 +0800 Subject: [PATCH 030/238] enh: optimize order by for partition by clause --- source/libs/planner/src/planLogicCreater.c | 23 ++++--- tests/script/tsim/query/partitionby.sim | 79 ++++++++++++++++++++-- 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b9ea5ba0cd..91a5e6ac93 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -37,19 +37,24 @@ typedef struct SRewriteExprCxt { int32_t errCode; SNodeList* pExprs; bool* pOutputs; + bool isPartitionBy; } SRewriteExprCxt; -static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { +static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartitionBy) { switch (pFunc->funcType) { case FUNCTION_TYPE_TBNAME: pCol->colType = COLUMN_TYPE_TBNAME; break; case FUNCTION_TYPE_WSTART: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_START; break; case FUNCTION_TYPE_WEND: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_END; break; case FUNCTION_TYPE_WDURATION: @@ -103,7 +108,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { - setColumnInfo((SFunctionNode*)pExpr, pCol); + setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; @@ -142,7 +147,8 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExpr(pExpr, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); if (TSDB_CODE_SUCCESS == cxt.errCode) { nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); @@ -170,7 +176,8 @@ static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList** static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause, SNodeList** pRewriteExprs) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; if (NULL != pRewriteExprs) { cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool)); if (NULL == cxt.pOutputs) { @@ -187,14 +194,14 @@ static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ES static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } diff --git a/tests/script/tsim/query/partitionby.sim b/tests/script/tsim/query/partitionby.sim index 8babd1aa8d..4657713780 100644 --- a/tests/script/tsim/query/partitionby.sim +++ b/tests/script/tsim/query/partitionby.sim @@ -24,16 +24,83 @@ sql use $db sql create table $mt1 (ts timestamp, f1 int) TAGS(tag1 int, tag2 binary(500)) sql create table tb0 using $mt1 tags(0, 'a'); sql create table tb1 using $mt1 tags(1, 'b'); -sql create table tb2 using $mt1 tags(1, 'a'); -sql create table tb3 using $mt1 tags(1, 'a'); -sql create table tb4 using $mt1 tags(3, 'b'); -sql create table tb5 using $mt1 tags(3, 'a'); -sql create table tb6 using $mt1 tags(3, 'b'); -sql create table tb7 using $mt1 tags(3, 'b'); +sql create table tb2 using $mt1 tags(2, 'a'); +sql create table tb3 using $mt1 tags(3, 'a'); +sql create table tb4 using $mt1 tags(4, 'b'); +sql create table tb5 using $mt1 tags(5, 'a'); +sql create table tb6 using $mt1 tags(6, 'b'); +sql create table tb7 using $mt1 tags(7, 'b'); sql select * from $mt1 partition by tag1,tag2 limit 1; if $rows != 0 then return -1 endi +sql insert into tb0 values ('2022-04-26 15:15:08', 1); +sql insert into tb1 values ('2022-04-26 15:15:07', 2); +sql insert into tb2 values ('2022-04-26 15:15:06', 3); +sql insert into tb3 values ('2022-04-26 15:15:05', 4); +sql insert into tb4 values ('2022-04-26 15:15:04', 5); +sql insert into tb5 values ('2022-04-26 15:15:03', 6); +sql insert into tb6 values ('2022-04-26 15:15:02', 7); +sql insert into tb7 values ('2022-04-26 15:15:01', 8); + +sql select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart; +if $rows != 8 then + return -1 +endi +if $data00 != @22-04-26 15:15:01.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data10 != @22-04-26 15:15:02.000@ then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data20 != @22-04-26 15:15:03.000@ then + return -1 +endi +if $data21 != 1 then + return -1 +endi +if $data30 != @22-04-26 15:15:04.000@ then + return -1 +endi +if $data31 != 1 then + return -1 +endi +if $data40 != @22-04-26 15:15:05.000@ then + return -1 +endi +if $data41 != 1 then + return -1 +endi +if $data50 != @22-04-26 15:15:06.000@ then + return -1 +endi +if $data51 != 1 then + return -1 +endi +if $data60 != @22-04-26 15:15:07.000@ then + return -1 +endi +if $data61 != 1 then + return -1 +endi +if $data70 != @22-04-26 15:15:08.000@ then + return -1 +endi +if $data71 != 1 then + return -1 +endi +sql select * from (select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart) order by ts; +sql select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart; +sql select * from (select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart) order by ts; +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 interval(1s) order by _wstart); +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 partition by tag1 interval(1s) order by _wstart); + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 4db88e8ffe49e5dbbb56caf25b51d157dc13042e Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 15:45:58 +0800 Subject: [PATCH 031/238] fix: fix ci comments --- source/libs/executor/src/joinoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 700a59ccd8..9dee58367c 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -438,7 +438,6 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { - tSimpleHashClear(pInfo->rightBuildTable); SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen); From 99b424b181d527a3ce368b199dcf8be0a142d70d Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 16:26:20 +0800 Subject: [PATCH 032/238] fix: add test case --- tests/script/output.txt | 6 ++ tests/script/sh/l2norm2.c | 71 ++++++++++++++++++++ tests/script/tsim/parser/join_manyblocks.sim | 8 +-- 3 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 tests/script/output.txt create mode 100644 tests/script/sh/l2norm2.c diff --git a/tests/script/output.txt b/tests/script/output.txt new file mode 100644 index 0000000000..0c7c386bbc --- /dev/null +++ b/tests/script/output.txt @@ -0,0 +1,6 @@ +[03/30 08:08:52.140115] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection +[03/30 08:08:52.140136] ERROR: insert test process failed +[03/30 08:09:02.860424] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection +[03/30 08:09:02.860445] ERROR: insert test process failed +[03/30 08:10:57.082603] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection +[03/30 08:10:57.082626] ERROR: insert test process failed diff --git a/tests/script/sh/l2norm2.c b/tests/script/sh/l2norm2.c new file mode 100644 index 0000000000..0739f3a23b --- /dev/null +++ b/tests/script/sh/l2norm2.c @@ -0,0 +1,71 @@ +#include +#include +#include +#include + +#include "taosudf.h" + +DLL_EXPORT int32_t l2norm2_init() { + return 0; +} + +DLL_EXPORT int32_t l2norm2_destroy() { + return 0; +} + +DLL_EXPORT int32_t l2norm2_start(SUdfInterBuf *buf) { + *(int64_t*)(buf->buf) = 0; + buf->bufLen = sizeof(double); + buf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t l2norm2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { + double sumSquares = *(double*)interBuf->buf; + int8_t numNotNull = 0; + for (int32_t i = 0; i < block->numOfCols; ++i) { + SUdfColumn* col = block->udfCols[i]; + if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || + col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } + for (int32_t i = 0; i < block->numOfCols; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { + SUdfColumn* col = block->udfCols[i]; + if (udfColDataIsNull(col, j)) { + continue; + } + switch (col->colMeta.type) { + case TSDB_DATA_TYPE_INT: { + char* cell = udfColDataGetData(col, j); + int32_t num = *(int32_t*)cell; + sumSquares += (double)num * num; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + char* cell = udfColDataGetData(col, j); + double num = *(double*)cell; + sumSquares += num * num; + break; + } + default: + break; + } + ++numNotNull; + } + } + + *(double*)(newInterBuf->buf) = sumSquares; + newInterBuf->bufLen = sizeof(double); + newInterBuf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t l2norm2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { + double sumSquares = *(double*)(buf->buf); + *(double*)(resultData->buf) = sumSquares; + resultData->bufLen = sizeof(double); + resultData->numOfResult = 1; + return 0; +} diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index a40a75f50c..7fd0df21b3 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -6,8 +6,8 @@ sql connect $dbPrefix = join_m_db $tbPrefix = join_tb $mtPrefix = join_mt -$tbNum = 3 -$rowNum = 2000 +$tbNum = 20 +$rowNum = 200 $totalNum = $tbNum * $rowNum print =============== join_manyBlocks.sim @@ -78,8 +78,8 @@ print ==============> td-3313 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; print $row -if $row != 6000 then - print expect 6000, actual: $row +if $row != 4000 then + print expect 4000, actual: $row return -1 endi From 3e02bb8faa96dfb7c23fe34357d47231e1033dd9 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 16:26:52 +0800 Subject: [PATCH 033/238] fix: remove files --- tests/script/output.txt | 6 ---- tests/script/sh/l2norm2.c | 71 --------------------------------------- 2 files changed, 77 deletions(-) delete mode 100644 tests/script/output.txt delete mode 100644 tests/script/sh/l2norm2.c diff --git a/tests/script/output.txt b/tests/script/output.txt deleted file mode 100644 index 0c7c386bbc..0000000000 --- a/tests/script/output.txt +++ /dev/null @@ -1,6 +0,0 @@ -[03/30 08:08:52.140115] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection -[03/30 08:08:52.140136] ERROR: insert test process failed -[03/30 08:09:02.860424] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection -[03/30 08:09:02.860445] ERROR: insert test process failed -[03/30 08:10:57.082603] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection -[03/30 08:10:57.082626] ERROR: insert test process failed diff --git a/tests/script/sh/l2norm2.c b/tests/script/sh/l2norm2.c deleted file mode 100644 index 0739f3a23b..0000000000 --- a/tests/script/sh/l2norm2.c +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include -#include - -#include "taosudf.h" - -DLL_EXPORT int32_t l2norm2_init() { - return 0; -} - -DLL_EXPORT int32_t l2norm2_destroy() { - return 0; -} - -DLL_EXPORT int32_t l2norm2_start(SUdfInterBuf *buf) { - *(int64_t*)(buf->buf) = 0; - buf->bufLen = sizeof(double); - buf->numOfResult = 1; - return 0; -} - -DLL_EXPORT int32_t l2norm2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { - double sumSquares = *(double*)interBuf->buf; - int8_t numNotNull = 0; - for (int32_t i = 0; i < block->numOfCols; ++i) { - SUdfColumn* col = block->udfCols[i]; - if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || - col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) { - return TSDB_CODE_UDF_INVALID_INPUT; - } - } - for (int32_t i = 0; i < block->numOfCols; ++i) { - for (int32_t j = 0; j < block->numOfRows; ++j) { - SUdfColumn* col = block->udfCols[i]; - if (udfColDataIsNull(col, j)) { - continue; - } - switch (col->colMeta.type) { - case TSDB_DATA_TYPE_INT: { - char* cell = udfColDataGetData(col, j); - int32_t num = *(int32_t*)cell; - sumSquares += (double)num * num; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - char* cell = udfColDataGetData(col, j); - double num = *(double*)cell; - sumSquares += num * num; - break; - } - default: - break; - } - ++numNotNull; - } - } - - *(double*)(newInterBuf->buf) = sumSquares; - newInterBuf->bufLen = sizeof(double); - newInterBuf->numOfResult = 1; - return 0; -} - -DLL_EXPORT int32_t l2norm2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { - double sumSquares = *(double*)(buf->buf); - *(double*)(resultData->buf) = sumSquares; - resultData->bufLen = sizeof(double); - resultData->numOfResult = 1; - return 0; -} From 79623860a43cdd9604e7e41037bdb5db6044d393 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 May 2023 17:41:01 +0800 Subject: [PATCH 034/238] fix: distinct wrong result issue --- source/libs/planner/src/planLogicCreater.c | 9 ++++-- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/groupby_distinct.sim | 30 ++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 tests/script/tsim/query/groupby_distinct.sim diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b9ea5ba0cd..16bd16238e 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -544,11 +544,16 @@ static SNode* createGroupingSetNode(SNode* pExpr) { return (SNode*)pGroupingSet; } -static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static EGroupAction getDistinctGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; } +static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { + return ((pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) && !pSelect->isDistinct) ? GROUP_ACTION_KEEP + : GROUP_ACTION_NONE; +} + static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; @@ -1152,7 +1157,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = GROUP_ACTION_CLEAR; + pAgg->node.groupAction = getDistinctGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e79cf77648..bcfb392bb4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -900,6 +900,7 @@ ,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim +,,y,script,./test.sh -f tsim/query/groupby_distinct.sim ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim diff --git a/tests/script/tsim/query/groupby_distinct.sim b/tests/script/tsim/query/groupby_distinct.sim new file mode 100644 index 0000000000..8b16bb1910 --- /dev/null +++ b/tests/script/tsim/query/groupby_distinct.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + + +sql drop database if exists db1; +sql create database db1; +sql use db1; + +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql insert into tba1 values ('2022-04-26 15:15:08', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:07', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:06', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:05', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:03', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:02', 1, "d"); +sql insert into tba1 values ('2022-04-26 15:15:01', 1, "d"); +sql select distinct avg(f1) as avgv from sta group by f2; +if $rows != 1 then + return -1 +endi +sql select distinct avg(f1) as avgv from sta group by f2 limit 1,10; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From c7e6883ac8e2f47a2514246abecd675781f88f91 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 18:10:33 +0800 Subject: [PATCH 035/238] fix:tmq ref err & vg error --- source/client/src/clientTmq.c | 101 +++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 20 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index b488af9ba1..be547c42b2 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -87,6 +87,7 @@ struct tmq_t { void* commitCbUserParam; // status + SRWLatch lock; int8_t status; int32_t epoch; #if 0 @@ -156,6 +157,7 @@ typedef struct { int8_t tmqRspType; int32_t epoch; // epoch can be used to guard the vgHandle int32_t vgId; + char topicName[TSDB_TOPIC_FNAME_LEN]; SMqClientVg* vgHandle; SMqClientTopic* topicHandle; uint64_t reqId; @@ -168,8 +170,8 @@ typedef struct { } SMqPollRspWrapper; typedef struct { - int64_t refId; - int32_t epoch; +// int64_t refId; +// int32_t epoch; tsem_t rspSem; int32_t rspErr; } SMqSubscribeCbParam; @@ -184,8 +186,9 @@ typedef struct { typedef struct { int64_t refId; int32_t epoch; - SMqClientVg* pVg; - SMqClientTopic* pTopic; + char topicName[TSDB_TOPIC_FNAME_LEN]; +// SMqClientVg* pVg; +// SMqClientTopic* pTopic; int32_t vgId; uint64_t requestId; // request id for debug purpose } SMqPollCbParam; @@ -709,8 +712,8 @@ static void generateTimedTask(int64_t refId, int32_t type) { *pTaskType = type; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); } - taosReleaseRef(tmqMgmt.rsetId, refId); } void tmqAssignAskEpTask(void* param, void* tmrId) { @@ -1037,6 +1040,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->commitCb = conf->commitCb; pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; + taosInitRWLatch(&pTmq->lock); pTmq->hbBgEnable = conf->hbBgEnable; @@ -1140,8 +1144,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SMqSubscribeCbParam param = { .rspErr = 0, - .refId = tmq->refId, - .epoch = tmq->epoch, +// .refId = tmq->refId, +// .epoch = tmq->epoch, }; if (tsem_init(¶m.rspSem, 0, 0) != 0) { @@ -1217,12 +1221,40 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para conf->commitCbUserParam = param; } +static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + for (int32_t j = 0; j < vgNumCur; j++) { + SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); + if(pVgCur->vgId == vgId){ + return pVgCur; + } + } + } + } + return NULL; +} + +static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + return pTopicCur; + } + } + return NULL; +} + int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; int64_t refId = pParam->refId; - SMqClientVg* pVg = pParam->pVg; - SMqClientTopic* pTopic = pParam->pTopic; +// SMqClientVg* pVg = pParam->pVg; +// SMqClientTopic* pTopic = pParam->pTopic; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { @@ -1303,11 +1335,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } pRspWrapper->tmqRspType = rspType; - pRspWrapper->vgHandle = pVg; - pRspWrapper->topicHandle = pTopic; +// pRspWrapper->vgHandle = pVg; +// pRspWrapper->topicHandle = pTopic; pRspWrapper->reqId = requestId; pRspWrapper->pEpset = pMsg->pEpSet; - pRspWrapper->vgId = pVg->vgId; + pRspWrapper->vgId = vgId; + strcpy(pRspWrapper->topicName, pParam->topicName); pMsg->pEpSet = NULL; if (rspType == TMQ_MSG_TYPE__POLL_RSP) { @@ -1351,7 +1384,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { CREATE_MSG_FAIL: if (epoch == tmq->epoch) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = getVgInfo(tmq, pParam->topicName, vgId); + if(pVg){ + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + taosWLockLatch(&tmq->lock); } tsem_post(&tmq->rspSem); @@ -1472,11 +1510,13 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) taosHashCleanup(pVgOffsetHashMap); + taosWLockLatch(&tmq->lock); // destroy current buffered existed topics info if (tmq->clientTopics) { taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo); } tmq->clientTopics = newTopics; + taosWUnLockLatch(&tmq->lock); int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; atomic_store_8(&tmq->status, flag); @@ -1492,7 +1532,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { if (tmq == NULL) { terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); +// pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -1652,8 +1692,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam->refId = pTmq->refId; pParam->epoch = pTmq->epoch; - pParam->pVg = pVg; // pVg may be released,fix it - pParam->pTopic = pTopic; +// pParam->pVg = pVg; // pVg may be released,fix it +// pParam->pTopic = pTopic; + strcpy(pParam->topicName, pTopic->topicName); pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; @@ -1779,8 +1820,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; if (pDataRsp->head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; - + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } // update the epset if (pollRspWrapper->pEpset != NULL) { SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset); @@ -1829,7 +1876,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } if(pollRspWrapper->metaRsp.rspOffset.type != 0){ // if offset is validate pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; } @@ -1849,7 +1903,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate pVg->currentOffset = pollRspWrapper->taosxRsp.rspOffset; } From 7db9f1e58ed471bcd22e5d692bd11c47835140d6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 18:21:45 +0800 Subject: [PATCH 036/238] fix:tmq ref err & vg error --- source/client/src/clientTmq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index be547c42b2..433b27d5d9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1269,8 +1269,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t vgId = pParam->vgId; uint64_t requestId = pParam->requestId; - taosMemoryFree(pParam); - if (code != 0) { if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); @@ -1314,6 +1312,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pParam); + return 0; } @@ -1379,6 +1379,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return 0; @@ -1394,6 +1395,7 @@ CREATE_MSG_FAIL: tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return -1; } From 50bdfef7b6e32134a928fc7e6cd0232b2163054e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 18:34:42 +0800 Subject: [PATCH 037/238] fix:tmq ref err & vg error --- source/client/src/clientTmq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 433b27d5d9..01667f6dc3 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1390,7 +1390,7 @@ CREATE_MSG_FAIL: if(pVg){ atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } - taosWLockLatch(&tmq->lock); + taosWUnLockLatch(&tmq->lock); } tsem_post(&tmq->rspSem); From bca1af3f5d7cd00d98096b1b0f5e052b8e8c0790 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 21:18:44 +0800 Subject: [PATCH 038/238] fix:change tmq subscribe try time to 60mins --- source/client/src/clientTmq.c | 4 ++-- source/dnode/vnode/src/tq/tqPush.c | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 01667f6dc3..c08fbd0adf 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1089,7 +1089,7 @@ _failed: } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { - const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most + const int32_t MAX_RETRY_COUNT = 120 * 60; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1186,7 +1186,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { if (retryCnt++ > MAX_RETRY_COUNT) { tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); - code = TSDB_CODE_TSC_INTERNAL_ERROR; + code = TSDB_CODE_MND_CONSUMER_NOT_READY; goto FAIL; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 50d94c5ed5..42c60e0007 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -73,8 +73,11 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) { STqHandle *pHandle = (STqHandle*)handle; int32_t vgId = TD_VID(pTq->pVnode); + if(taosHashGetSize(pTq->pPushMgr) <= 0) { + return 0; + } int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey)); - tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); + tqDebug("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); if(pHandle->msg != NULL) { tqPushDataRsp(pTq, pHandle); From d4b0e102af08e156ca8bf61fa091b38d21ca3933 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 May 2023 08:55:59 +0800 Subject: [PATCH 039/238] fix:do some internal refactor. --- include/common/ttime.h | 2 +- source/common/src/ttime.c | 2 -- source/libs/executor/src/executil.c | 2 +- source/libs/parser/src/parTranslater.c | 8 ++++---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 69a071d59f..7bbc6a8e56 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -23,7 +23,7 @@ extern "C" { #endif -#define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') +#define IS_CALENDAR_TIME_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') #define TIME_UNIT_NANOSECOND 'b' #define TIME_UNIT_MICROSECOND 'u' diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index de6a18f7c8..165a4ff95b 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -709,8 +709,6 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati return getDuration(*duration, *unit, duration, timePrecision); } -#define IS_CALENDAR_TIME_DURATION(_d) (((_d) == 'n') || ((_d) == 'y')) - int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { return t; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 91f384ce6a..b4c5a90be3 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1759,7 +1759,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->slidingUnit != 'n' && pInterval->slidingUnit != 'y') { + if (!IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { tw->skey += pInterval->sliding * factor; tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0d052846f7..c61c4fe4d4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3004,7 +3004,7 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey); int64_t intervalRange = 0; - if (TIME_IS_VAR_DURATION(pInterval->unit)) { + if (IS_CALENDAR_TIME_DURATION(pInterval->unit)) { int64_t f = 1; if (pInterval->unit == 'n') { f = 30LL * MILLISECOND_PER_DAY; @@ -3072,7 +3072,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = TIME_IS_VAR_DURATION(pInter->unit); + bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit); if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, getPrecisionStr(precision)); @@ -3086,7 +3086,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->unit == 'n' && pOffset->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter; + bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { @@ -3098,7 +3098,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* const static int32_t INTERVAL_SLIDING_FACTOR = 100; SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (TIME_IS_VAR_DURATION(pSliding->unit)) { + if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || From 93c4e3fe70a9a989d8b96a457b5240147190b4a7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 May 2023 09:12:16 +0800 Subject: [PATCH 040/238] other: remove empty files. --- source/libs/executor/inc/executorimpl.h | 0 source/libs/executor/src/executorimpl.c | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 source/libs/executor/inc/executorimpl.h delete mode 100644 source/libs/executor/src/executorimpl.c diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c deleted file mode 100644 index e69de29bb2..0000000000 From e882601b67f900dde0790d1e55628dfd7b86e4c5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 May 2023 09:15:48 +0800 Subject: [PATCH 041/238] fix: last table scan issue --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 21b59e376c..4acc784aee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3112,6 +3112,10 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) { + // reset the index in last block when handing a new file + doCleanupTableScanInfo(pScanInfo); + pStatus->mapDataCleaned = true; + bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { return TSDB_CODE_SUCCESS; From 7218a034870adc3f336b4672b88e12639b20daa2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 May 2023 11:16:31 +0800 Subject: [PATCH 042/238] fix: result block ensure capacity when finalize may produce serveral rows --- source/libs/executor/src/executorInt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 1a14d0e78a..303be3d2cd 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -715,7 +715,8 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR pCtx[j].resultInfo->numOfRes = pRow->numOfRows; } } - + + blockDataEnsureCapacity(pBlock, pBlock->info.rows + pCtx[j].resultInfo->numOfRes); int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); if (TAOS_FAILED(code)) { qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); From 6a1eb62dd44956c6cb4a7453c83eb96687e1cefc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 May 2023 11:26:40 +0800 Subject: [PATCH 043/238] fix: distinct group issue --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 16bd16238e..1ec0abe8df 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1157,7 +1157,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = getDistinctGroupAction(pCxt, pSelect); + pAgg->node.groupAction = GROUP_ACTION_CLEAR;//getDistinctGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; From 238254e49ee83723dd274bae5a66d7fe9994d5df Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 14:06:43 +0800 Subject: [PATCH 044/238] fix:add task status check in doQueueScan & checkout consumer in tqProcessPollReq --- source/dnode/vnode/src/tq/tq.c | 22 +++++++++--------- source/libs/executor/src/scanoperator.c | 30 +------------------------ 2 files changed, 11 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e074a2d8a6..41b43026ea 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -358,9 +358,19 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } + // 2. check re-balance status + if (pHandle->consumerId != consumerId) { + tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosWUnLockLatch(&pTq->lock); + return -1; + } + bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; } @@ -370,17 +380,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } - // 2. check re-balance status - if (pHandle->consumerId != consumerId) { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, - consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - code = -1; - goto end; - } - - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); - // 3. update the epoch value int32_t savedEpoch = pHandle->epoch; if (savedEpoch < reqEpoch) { @@ -396,7 +395,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); -end: tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7d3165237e..638662aeb5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1665,37 +1665,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("start to exec queue scan, %s", id); -#if 0 - if (pTaskInfo->streamInfo.submit.msgStr != NULL) { - if (pInfo->tqReader->msg.msgStr == NULL) { - SPackedData submit = pTaskInfo->streamInfo.submit; - if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { - qError("submit msg messed up when initing stream submit block %p", submit.msgStr); - return NULL; - } - } - - blockDataCleanup(pInfo->pRes); - SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - - while (tqNextBlockImpl(pInfo->tqReader)) { - int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL); - if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { - continue; - } - - setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true); - - if (pBlockInfo->rows > 0) { - return pInfo->pRes; - } - } - - pInfo->tqReader->msg = (SPackedData){0}; - pTaskInfo->streamInfo.submit = (SPackedData){0}; + if (isTaskKilled(pTaskInfo)) { return NULL; } -#endif if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); From f36b0be17d0c91dcd45941bd5d753d39293e6856 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 May 2023 14:32:03 +0800 Subject: [PATCH 045/238] fix: join eq conditions not only for tag --- include/libs/nodes/plannodes.h | 16 +++---- source/libs/executor/src/joinoperator.c | 56 +++++++++++----------- source/libs/nodes/src/nodesCloneFuncs.c | 4 +- source/libs/nodes/src/nodesCodeFuncs.c | 12 ++--- source/libs/nodes/src/nodesMsgFuncs.c | 8 ++-- source/libs/nodes/src/nodesUtilFuncs.c | 14 +++--- source/libs/planner/src/planOptimizer.c | 33 ++++++------- source/libs/planner/src/planPhysiCreater.c | 6 +-- 8 files changed, 73 insertions(+), 76 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 0e235191b1..f559ba2b51 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,7 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; - SNode* pTagEqualConditions; + SNode* pEqualOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -406,7 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; - SNode* pTagEqualCondtions; + SNode* pEqualOnCondtions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { @@ -448,7 +448,7 @@ typedef struct SMergePhysiNode { bool groupSort; } SMergePhysiNode; -typedef struct SWinodwPhysiNode { +typedef struct SWindowPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; @@ -461,10 +461,10 @@ typedef struct SWinodwPhysiNode { EOrder inputTsOrder; EOrder outputTsOrder; bool mergeDataBlock; -} SWinodwPhysiNode; +} SWindowPhysiNode; typedef struct SIntervalPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t interval; int64_t offset; int64_t sliding; @@ -497,7 +497,7 @@ typedef struct SMultiTableIntervalPhysiNode { } SMultiTableIntervalPhysiNode; typedef struct SSessionWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t gap; } SSessionWinodwPhysiNode; @@ -506,14 +506,14 @@ typedef SSessionWinodwPhysiNode SStreamSemiSessionWinodwPhysiNode; typedef SSessionWinodwPhysiNode SStreamFinalSessionWinodwPhysiNode; typedef struct SStateWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStateKey; } SStateWinodwPhysiNode; typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; typedef struct SEventWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStartCond; SNode* pEndCond; } SEventWinodwPhysiNode; diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 9dee58367c..46c6b24295 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -52,15 +52,15 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; - SNode* pTagEqualConditions; + SNode* pEqualOnConditions; - SArray* leftTagCols; - char* leftTagKeyBuf; - int32_t leftTagKeyLen; + SArray* leftEqOnCondCols; + char* leftEqOnCondKeyBuf; + int32_t leftEqOnCondKeyLen; - SArray* rightTagCols; - char* rightTagKeyBuf; - int32_t rightTagKeyLen; + SArray* rightEqOnCondCols; + char* rightEqOnCondKeyBuf; + int32_t rightEqOnCondKeyLen; SSHashObj* rightBuildTable; SJoinRowCtx rowCtx; @@ -104,7 +104,7 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } -static void extractTagEqualColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, +static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, SColumn* pLeft, SColumn* pRight) { SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft; SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight; @@ -125,7 +125,7 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD SNode* pNode = NULL; FOREACH(pNode, ((SLogicConditionNode*)pTagEqualNode)->pParameterList) { SOperatorNode* pOperNode = (SOperatorNode*)pNode; - extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); taosArrayPush(rightTagEqCols, &right); } @@ -134,7 +134,7 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD if (nodeType(pTagEqualNode) == QUERY_NODE_OPERATOR) { SOperatorNode* pOperNode = (SOperatorNode*)pTagEqualNode; - extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); taosArrayPush(rightTagEqCols, &right); } @@ -259,13 +259,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pInfo->pTagEqualConditions = pJoinNode->pTagEqualCondtions; - if (pInfo->pTagEqualConditions != NULL) { - pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); - pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); - extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); - initTagColskeyBuf(&pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); - initTagColskeyBuf(&pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + pInfo->pEqualOnConditions = pJoinNode->pEqualOnCondtions; + if (pInfo->pEqualOnConditions != NULL) { + pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + extractTagEqualCondCols(pInfo, pDownstream, pInfo->pEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); + initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols); + initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->rightBuildTable = tSimpleHashInit(256, hashFn); } @@ -309,13 +309,13 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; - if (pJoinOperator->pTagEqualConditions != NULL) { + if (pJoinOperator->pEqualOnConditions != NULL) { mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); - taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); - taosArrayDestroy(pJoinOperator->rightTagCols); + taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->rightEqOnCondCols); - taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); - taosArrayDestroy(pJoinOperator->leftTagCols); + taosMemoryFreeClear(pJoinOperator->leftEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->leftEqOnCondCols); } nodesDestroyNode(pJoinOperator->pCondAfterMerge); @@ -439,12 +439,12 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); - int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); - SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen); + int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightEqOnCondCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightEqOnCondKeyBuf); + SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen); if (!ppRows) { SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); taosArrayPush(rows, rightRow); - tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); + tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen, &rows, POINTER_BYTES); } else { taosArrayPush(*ppRows, rightRow); } @@ -466,8 +466,8 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SArray* pRightRows = NULL; if (useBuildTableTSRange) { - int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftTagCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftTagKeyBuf); - SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftTagKeyBuf, keyLen); + int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftEqOnCondCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftEqOnCondKeyBuf); + SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftEqOnCondKeyBuf, keyLen); if (!ppRightRows) { continue; } @@ -567,7 +567,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); - if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + if (pJoinInfo->pEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 91db527a02..c2771e0005 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,7 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); - CLONE_NODE_FIELD(pTagEqualConditions); + CLONE_NODE_FIELD(pEqualOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; @@ -588,7 +588,7 @@ static int32_t physiSysTableScanCopy(const SSystemTableScanPhysiNode* pSrc, SSys return TSDB_CODE_SUCCESS; } -static int32_t physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* pDst) { +static int32_t physiWindowCopy(const SWindowPhysiNode* pSrc, SWindowPhysiNode* pDst) { COPY_BASE_OBJECT_FIELD(node, physiNodeCopy); CLONE_NODE_LIST_FIELD(pExprs); CLONE_NODE_LIST_FIELD(pFuncs); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b4e2d95e26..d16cd79e97 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1432,7 +1432,7 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pTagEqualConditions); + code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pEqualOnConditions); } return code; } @@ -1451,7 +1451,7 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pTagEqualConditions); + code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pEqualOnConditions); } return code; } @@ -1905,7 +1905,7 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pTagEqualCondtions); + code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pEqualOnCondtions); } return code; } @@ -1930,7 +1930,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pTagEqualCondtions); + code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pEqualOnCondtions); } return code; } @@ -2135,7 +2135,7 @@ static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { @@ -2176,7 +2176,7 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { } static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 45cebb4559..59b027d5ed 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2341,7 +2341,7 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pTagEqualCondtions); + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pEqualOnCondtions); } return code; } @@ -2372,7 +2372,7 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: - code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagEqualCondtions); + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEqualOnCondtions); break; default: break; @@ -2639,7 +2639,7 @@ enum { }; static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); if (TSDB_CODE_SUCCESS == code) { @@ -2680,7 +2680,7 @@ static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { } static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = TSDB_CODE_SUCCESS; STlv* pTlv = NULL; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 67747497db..81b429e169 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -599,7 +599,7 @@ static void destroyPhysiNode(SPhysiNode* pNode) { nodesDestroyNode(pNode->pSlimit); } -static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { +static void destroyWinodwPhysiNode(SWindowPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pExprs); nodesDestroyList(pNode->pFuncs); @@ -1072,7 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); - nodesDestroyNode(pLogicNode->pTagEqualConditions); + nodesDestroyNode(pLogicNode->pEqualOnConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1205,7 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); - nodesDestroyNode(pPhyNode->pTagEqualCondtions); + nodesDestroyNode(pPhyNode->pEqualOnCondtions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { @@ -1243,7 +1243,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: { @@ -1259,19 +1259,19 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: { SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStateKey); break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStartCond); nodesDestroyNode(pPhyNode->pEndCond); break; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 5be67389c8..1107389df9 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -740,18 +740,15 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin return code; } -static bool pushDownCondOptIsTag(SNode* pNode, SNodeList* pTableCols) { +static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) { if (QUERY_NODE_COLUMN != nodeType(pNode)) { return false; } SColumnNode* pCol = (SColumnNode*)pNode; - if (COLUMN_TYPE_TAG != pCol->colType) { - return false; - } return pushDownCondOptBelongThisTable(pNode, pTableCols); } -static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } @@ -770,22 +767,22 @@ static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { } SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; - if (pushDownCondOptIsTag(pOper->pLeft, pLeftCols)) { - return pushDownCondOptIsTag(pOper->pRight, pRightCols); - } else if (pushDownCondOptIsTag(pOper->pLeft, pRightCols)) { - return pushDownCondOptIsTag(pOper->pRight, pLeftCols); + if (pushDownCondOptIsTableColumn(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsTableColumn(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pLeftCols); } return false; } -static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); int32_t code = TSDB_CODE_SUCCESS; SNodeList* pTagEqualConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - if (pushDownCondOptIsTagEqualCond(pJoin, pCond)) { + if (pushDownCondOptIsEqualOnCond(pJoin, pCond)) { code = nodesListMakeAppend(&pTagEqualConds, nodesCloneNode(pCond)); } } @@ -796,7 +793,7 @@ static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin } if (TSDB_CODE_SUCCESS == code) { - pJoin->pTagEqualConditions = pTempTagEqCond; + pJoin->pEqualOnConditions = pTempTagEqCond; return TSDB_CODE_SUCCESS; } else { nodesDestroyList(pTagEqualConds); @@ -805,18 +802,18 @@ static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin return TSDB_CODE_SUCCESS; } -static int32_t pushDownCondOptJoinExtractTagEqualCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { - pJoin->pTagEqualConditions = NULL; + pJoin->pEqualOnConditions = NULL; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { - return pushDownCondOptJoinExtractTagEqualLogicCond(pJoin); + return pushDownCondOptJoinExtractEqualOnLogicCond(pJoin); } - if (pushDownCondOptIsTagEqualCond(pJoin, pJoin->pOnConditions)) { - pJoin->pTagEqualConditions = nodesCloneNode(pJoin->pOnConditions); + if (pushDownCondOptIsEqualOnCond(pJoin, pJoin->pOnConditions)) { + pJoin->pEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); } return TSDB_CODE_SUCCESS; @@ -857,7 +854,7 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p } if (TSDB_CODE_SUCCESS == code) { - code = pushDownCondOptJoinExtractTagEqualCond(pCxt, pJoin); + code = pushDownCondOptJoinExtractEqualOnCond(pCxt, pJoin); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e07c6ebcfe..a13ca6cba9 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,8 +705,8 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } - if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pTagEqualConditions) { - code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pTagEqualConditions, &pJoin->pTagEqualCondtions); + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pEqualOnConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pEqualOnConditions, &pJoin->pEqualOnCondtions); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); @@ -1150,7 +1150,7 @@ static int32_t createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNo } } -static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWinodwPhysiNode* pWindow, +static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowPhysiNode* pWindow, SWindowLogicNode* pWindowLogicNode) { pWindow->triggerType = pWindowLogicNode->triggerType; pWindow->watermark = pWindowLogicNode->watermark; From f7ab8dabf086ef3740040635059aa36627bc1525 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 15:14:39 +0800 Subject: [PATCH 046/238] fix:[TD-23972] push subscribe msg to vnode even though consumer not change --- source/dnode/mnode/impl/src/mndSubscribe.c | 21 ++++++++++++---- source/dnode/vnode/src/tq/tq.c | 28 ++++++++++++---------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 2ae4ede25a..e62102fa77 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -133,10 +133,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg) { - if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { - terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; - return -1; - } +// if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { +// terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; +// return -1; +// } void *buf; int32_t tlen; @@ -269,6 +269,18 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { } } +static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ + for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ + SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); + SMqRebOutputVg outputVg = { + .oldConsumerId = pConsumerEp->consumerId, + .newConsumerId = pConsumerEp->consumerId, + .pVgEp = pVgEp, + }; + taosArrayPush(pOutput->rebVgs, &outputVg); + } +} + static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, int32_t imbConsumerNum) { const char *pSubKey = pOutput->pSub->key; @@ -318,6 +330,7 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas } } } + putNoTransferToOutput(pOutput, pConsumerEp); } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 41b43026ea..6f619e9bbe 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -569,20 +569,22 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); // atomic_store_32(&pHandle->epoch, 0); - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } - // remove if it has been register in the push manager, and return one empty block to consumer - taosWLockLatch(&pTq->lock); - tqUnregisterPushHandle(pTq, pHandle); - taosWUnLockLatch(&pTq->lock); - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } + + // kill executing task + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + } + + taosWLockLatch(&pTq->lock); + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } + // remove if it has been register in the push manager, and return one empty block to consumer + tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } end: From f9da4abdcb495d0d9c75e4a6e9f14da47621a070 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 May 2023 15:38:52 +0800 Subject: [PATCH 047/238] fix: change more variable names --- include/libs/nodes/plannodes.h | 4 ++-- source/libs/executor/src/joinoperator.c | 22 +++++++++++----------- source/libs/nodes/src/nodesCloneFuncs.c | 2 +- source/libs/nodes/src/nodesCodeFuncs.c | 12 ++++++------ source/libs/nodes/src/nodesMsgFuncs.c | 4 ++-- source/libs/nodes/src/nodesUtilFuncs.c | 4 ++-- source/libs/planner/src/planOptimizer.c | 14 +++++++------- source/libs/planner/src/planPhysiCreater.c | 4 ++-- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f559ba2b51..a27237cab3 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,7 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; - SNode* pEqualOnConditions; + SNode* pColEqualOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -406,7 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; - SNode* pEqualOnCondtions; + SNode* pColEqualOnConditions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 46c6b24295..442f8162ed 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -52,7 +52,7 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; - SNode* pEqualOnConditions; + SNode* pColEqualOnConditions; SArray* leftEqOnCondCols; char* leftEqOnCondKeyBuf; @@ -117,13 +117,13 @@ static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorIn } } -static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pTagEqualNode, +static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pEqualOnCondNode, SArray* leftTagEqCols, SArray* rightTagEqCols) { SColumn left = {0}; SColumn right = {0}; - if (nodeType(pTagEqualNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pTagEqualNode)->condType == LOGIC_COND_TYPE_AND) { + if (nodeType(pEqualOnCondNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pEqualOnCondNode)->condType == LOGIC_COND_TYPE_AND) { SNode* pNode = NULL; - FOREACH(pNode, ((SLogicConditionNode*)pTagEqualNode)->pParameterList) { + FOREACH(pNode, ((SLogicConditionNode*)pEqualOnCondNode)->pParameterList) { SOperatorNode* pOperNode = (SOperatorNode*)pNode; extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); @@ -132,8 +132,8 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD return; } - if (nodeType(pTagEqualNode) == QUERY_NODE_OPERATOR) { - SOperatorNode* pOperNode = (SOperatorNode*)pTagEqualNode; + if (nodeType(pEqualOnCondNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pOperNode = (SOperatorNode*)pEqualOnCondNode; extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); taosArrayPush(rightTagEqCols, &right); @@ -259,11 +259,11 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pInfo->pEqualOnConditions = pJoinNode->pEqualOnCondtions; - if (pInfo->pEqualOnConditions != NULL) { + pInfo->pColEqualOnConditions = pJoinNode->pColEqualOnConditions; + if (pInfo->pColEqualOnConditions != NULL) { pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); - extractTagEqualCondCols(pInfo, pDownstream, pInfo->pEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); + extractEqualOnCondCols(pInfo, pDownstream, pInfo->pColEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols); initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -309,7 +309,7 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; - if (pJoinOperator->pEqualOnConditions != NULL) { + if (pJoinOperator->pColEqualOnConditions != NULL) { mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf); taosArrayDestroy(pJoinOperator->rightEqOnCondCols); @@ -567,7 +567,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); - if (pJoinInfo->pEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + if (pJoinInfo->pColEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index c2771e0005..f18e666741 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,7 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); - CLONE_NODE_FIELD(pEqualOnConditions); + CLONE_NODE_FIELD(pColEqualOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index d16cd79e97..f061fe60d2 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1416,7 +1416,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanMergeCondition = "MergeConditions"; -static const char* jkJoinLogicPlanTagEqualConditions = "TagEqualConditions"; +static const char* jkJoinLogicPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; @@ -1432,7 +1432,7 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pEqualOnConditions); + code = tjsonAddObject(pJson, jkJoinLogicPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); } return code; } @@ -1451,7 +1451,7 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pEqualOnConditions); + code = jsonToNodeObject(pJson, jkJoinLogicPlanColEqualOnConditions, &pNode->pColEqualOnConditions); } return code; } @@ -1883,7 +1883,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition"; static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; -static const char* jkJoinPhysiPlanTagEqualConditions = "TagEqualConditions"; +static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; @@ -1905,7 +1905,7 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pEqualOnCondtions); + code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); } return code; } @@ -1930,7 +1930,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pEqualOnCondtions); + code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqualOnConditions); } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 59b027d5ed..e200b5dac5 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2341,7 +2341,7 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pEqualOnCondtions); + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqualOnConditions); } return code; } @@ -2372,7 +2372,7 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: - code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEqualOnCondtions); + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqualOnConditions); break; default: break; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 81b429e169..830d5886f4 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1072,7 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); - nodesDestroyNode(pLogicNode->pEqualOnConditions); + nodesDestroyNode(pLogicNode->pColEqualOnConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1205,7 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); - nodesDestroyNode(pPhyNode->pEqualOnCondtions); + nodesDestroyNode(pPhyNode->pColEqualOnConditions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 1107389df9..647a55a495 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -779,24 +779,24 @@ static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); int32_t code = TSDB_CODE_SUCCESS; - SNodeList* pTagEqualConds = NULL; + SNodeList* pEqualOnConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { if (pushDownCondOptIsEqualOnCond(pJoin, pCond)) { - code = nodesListMakeAppend(&pTagEqualConds, nodesCloneNode(pCond)); + code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond)); } } SNode* pTempTagEqCond = NULL; if (TSDB_CODE_SUCCESS == code) { - code = nodesMergeConds(&pTempTagEqCond, &pTagEqualConds); + code = nodesMergeConds(&pTempTagEqCond, &pEqualOnConds); } if (TSDB_CODE_SUCCESS == code) { - pJoin->pEqualOnConditions = pTempTagEqCond; + pJoin->pColEqualOnConditions = pTempTagEqCond; return TSDB_CODE_SUCCESS; } else { - nodesDestroyList(pTagEqualConds); + nodesDestroyList(pEqualOnConds); return TSDB_CODE_PLAN_INTERNAL_ERROR; } return TSDB_CODE_SUCCESS; @@ -804,7 +804,7 @@ static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { - pJoin->pEqualOnConditions = NULL; + pJoin->pColEqualOnConditions = NULL; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && @@ -813,7 +813,7 @@ static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJo } if (pushDownCondOptIsEqualOnCond(pJoin, pJoin->pOnConditions)) { - pJoin->pEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); + pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index a13ca6cba9..f11f1244b4 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,8 +705,8 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } - if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pEqualOnConditions) { - code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pEqualOnConditions, &pJoin->pEqualOnCondtions); + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqualOnConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqualOnConditions, &pJoin->pColEqualOnConditions); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); From d77eefdc93c7d6584eb033fb588b45714db66433 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 May 2023 15:43:31 +0800 Subject: [PATCH 048/238] fix: change more variable names --- source/libs/planner/src/planOptimizer.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 647a55a495..80beab3038 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -748,7 +748,7 @@ static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) { return pushDownCondOptBelongThisTable(pNode, pTableCols); } -static bool pushDownCondOptIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptIsColEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } @@ -775,14 +775,14 @@ static bool pushDownCondOptIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { return false; } -static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractColEqualOnLogicCond(SJoinLogicNode* pJoin) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); int32_t code = TSDB_CODE_SUCCESS; SNodeList* pEqualOnConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - if (pushDownCondOptIsEqualOnCond(pJoin, pCond)) { + if (pushDownCondOptIsColEqualOnCond(pJoin, pCond)) { code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond)); } } @@ -802,17 +802,17 @@ static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) return TSDB_CODE_SUCCESS; } -static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractColEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { pJoin->pColEqualOnConditions = NULL; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { - return pushDownCondOptJoinExtractEqualOnLogicCond(pJoin); + return pushDownCondOptJoinExtractColEqualOnLogicCond(pJoin); } - if (pushDownCondOptIsEqualOnCond(pJoin, pJoin->pOnConditions)) { + if (pushDownCondOptIsColEqualOnCond(pJoin, pJoin->pOnConditions)) { pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); } @@ -854,7 +854,7 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p } if (TSDB_CODE_SUCCESS == code) { - code = pushDownCondOptJoinExtractEqualOnCond(pCxt, pJoin); + code = pushDownCondOptJoinExtractColEqualOnCond(pCxt, pJoin); } if (TSDB_CODE_SUCCESS == code) { From be35194d169e77e97f524ff9e48e3c6c701fa58e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 May 2023 15:54:46 +0800 Subject: [PATCH 049/238] fix(query): add null check. --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index a196c2d398..8f168e67a8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -225,6 +225,9 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, void* tsdbCacherowsReaderClose(void* pReader) { SCacheRowsReader* p = pReader; + if (p == NULL) { + return NULL; + } if (p->pSchema != NULL) { for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) { From 6e6853413c0e4ce57dcd9355628cea2dc72b2fac Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 May 2023 16:48:05 +0800 Subject: [PATCH 050/238] fix: db cfg memory issue --- source/dnode/mnode/impl/src/mndDb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e93b06fdea..0f7e3de2f5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -929,7 +929,7 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->walRetentionSize = pDb->cfg.walRetentionSize; cfgRsp->walSegmentSize = pDb->cfg.walSegmentSize; cfgRsp->numOfRetensions = pDb->cfg.numOfRetensions; - cfgRsp->pRetensions = pDb->cfg.pRetensions; + cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); cfgRsp->schemaless = pDb->cfg.schemaless; cfgRsp->sstTrigger = pDb->cfg.sstTrigger; } @@ -972,6 +972,8 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { _OVER: + tFreeSDbCfgRsp(&cfgRsp); + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } From 3dbbe6309248c10e28cd7c2097f9d61f04d82b35 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 19 May 2023 16:50:27 +0800 Subject: [PATCH 051/238] fix: select last(*) ..group by tag return wrong rows of data --- source/libs/planner/src/planOptimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 5be67389c8..5d0757883c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2377,7 +2377,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (NULL != cxt.pLastCols) { cxt.doAgg = false; lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols); - NODES_DESTORY_LIST(pScan->pScanPseudoCols); + nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt); lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols); nodesClearList(cxt.pLastCols); } From ff2b545b27afa6f28f704d896910c04e727d19c8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 17:16:47 +0800 Subject: [PATCH 052/238] fix:set task status killed when vnode receive subscribe msg from mnode --- include/libs/executor/executor.h | 2 ++ source/dnode/vnode/src/tq/tq.c | 4 ++-- source/libs/executor/src/executor.c | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index b7e6c42e3b..2598b5c28c 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,6 +90,8 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); + int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); // todo refactor diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6f619e9bbe..9e43c4a944 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -394,7 +394,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); - + qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; @@ -574,7 +574,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // kill executing task qTaskInfo_t pTaskInfo = pHandle->execHandle.task; if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); } taosWLockLatch(&pTq->lock); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index c4a56d78ae..eea542e042 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,6 +180,11 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { + SExecTaskInfo* pTaskInfo = tinfo; + pTaskInfo->code = code; +} + int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { return TSDB_CODE_APP_ERROR; From c7c255e967aa250741bcb839f329634a99834b29 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 18:06:05 +0800 Subject: [PATCH 053/238] fix:set task status killed when vnode receive subscribe msg from mnode --- source/dnode/vnode/src/tq/tq.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9e43c4a944..b1e412d5ec 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -370,6 +370,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); + qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; @@ -394,7 +395,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); - qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; @@ -571,15 +571,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // atomic_store_32(&pHandle->epoch, 0); } - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); - } - taosWLockLatch(&pTq->lock); - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); + // kill executing task + if(tqIsHandleExec(pHandle)) { + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); + } + + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); From 33d6df8717875f1f2001d8ea8dde8339beee279d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 19:11:49 +0800 Subject: [PATCH 054/238] fix:set task status killed when vnode receive subscribe msg from mnode --- source/dnode/vnode/src/tq/tq.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b1e412d5ec..d9961202b1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -367,6 +367,20 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } + // 3. update the epoch value + int32_t savedEpoch = pHandle->epoch; + if (savedEpoch <= reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, + reqEpoch); + pHandle->epoch = reqEpoch; + }else { + tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", + consumerId, TD_VID(pTq->pVnode), req.subKey, savedEpoch, reqEpoch); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosWUnLockLatch(&pTq->lock); + return -1; + } + bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); @@ -381,14 +395,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } - // 3. update the epoch value - int32_t savedEpoch = pHandle->epoch; - if (savedEpoch < reqEpoch) { - tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, - reqEpoch); - pHandle->epoch = reqEpoch; - } - char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, @@ -396,6 +402,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); tqSetHandleIdle(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; } @@ -561,17 +568,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { + taosWLockLatch(&pTq->lock); + if (pHandle->consumerId == req.newConsumerId) { // do nothing tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs, should not reach here", req.vgId, req.newConsumerId); -// atomic_add_fetch_32(&pHandle->epoch, 1); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); -// atomic_store_32(&pHandle->epoch, 0); } + atomic_add_fetch_32(&pHandle->epoch, 1); - taosWLockLatch(&pTq->lock); // kill executing task if(tqIsHandleExec(pHandle)) { qTaskInfo_t pTaskInfo = pHandle->execHandle.task; From 1ddbdad33cf487e94d2edf6a3138324f123723e5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 21:40:33 +0800 Subject: [PATCH 055/238] fix:set task status killed when vnode receive subscribe msg from mnode --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d9961202b1..da366c1610 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -362,7 +362,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (pHandle->consumerId != consumerId) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + terrno = TSDB_CODE_TMQ_INVALID_MSG; taosWUnLockLatch(&pTq->lock); return -1; } From f8de38e5327ca92025c6f7a00fa6e02c08b6e710 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 22:52:58 +0800 Subject: [PATCH 056/238] fix:core dump in tsdbreader is null when tsdbreader is closed by subscribe msg and current offset equal to saved --- source/libs/executor/src/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index eea542e042..7d251fb074 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1080,7 +1080,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset) && pOffset->type != TMQ_OFFSET__SNAPSHOT_DATA) { return 0; } From 04858fae22b4b2f0fa6207b56e84dab0fc679ea6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 May 2023 12:20:54 +0800 Subject: [PATCH 057/238] fix:tsdbreader is free by mistake --- include/libs/executor/executor.h | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/dnode/vnode/src/tq/tq.c | 8 ++++---- source/dnode/vnode/src/tq/tqScan.c | 12 ++++++++---- source/libs/executor/src/executor.c | 10 +++++----- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 2598b5c28c..91db78e9cd 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,7 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index a318b9886e..922a85c094 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -75,7 +75,7 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, terrstr(code)); + dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, tstrerror(code)); vmSendRsp(pMsg, code); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index da366c1610..ce52277d71 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -384,7 +384,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); - qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); +// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; @@ -586,9 +586,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } +// if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { +// qStreamCloseTsdbReader(pTaskInfo); +// } } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index e4b2fa8821..2f470e2221 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -84,8 +84,10 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs qStreamSetOpen(task); tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); - if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { - tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); + code = qExecTask(task, &pDataBlock, &ts); + if (code != TSDB_CODE_SUCCESS) { + tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code)); + terrno = code; return -1; } @@ -128,8 +130,10 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("tmqsnap task start to execute"); - if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + int code = qExecTask(task, &pDataBlock, &ts); + if (code != 0) { + tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, tstrerror(code)); + terrno = code; return -1; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7d251fb074..a73deffa52 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,10 +180,10 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { - SExecTaskInfo* pTaskInfo = tinfo; - pTaskInfo->code = code; -} +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { +// SExecTaskInfo* pTaskInfo = tinfo; +// pTaskInfo->code = code; +//} int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { @@ -1080,7 +1080,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset) && pOffset->type != TMQ_OFFSET__SNAPSHOT_DATA) { + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } From 0b12224021345631b32ee889baddfaf13efcab28 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 May 2023 14:01:12 +0800 Subject: [PATCH 058/238] test: add schemless insert case --- .../system-test/eco-system/schemaless/insert.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/system-test/eco-system/schemaless/insert.py b/tests/system-test/eco-system/schemaless/insert.py index def70748d7..901196ebfd 100644 --- a/tests/system-test/eco-system/schemaless/insert.py +++ b/tests/system-test/eco-system/schemaless/insert.py @@ -104,34 +104,38 @@ class TDTestCase: tdLog.info(f" exe failed. i={i} {sql}") traceback.print_exc() + def genTags(self, i): + tags = f"t1={i},t2=abc,t3=work" + return tags # change table schema def schemaless_insert(self, change_cnt): # init ts = 1683194263000 for i in range(change_cnt): - t1 = i % 1000 index = int(i/10000) % 600 cols = self.genCols("c", 5, index, False) - tags = f"t1={t1},t2=abc,t3=work" - sql = f'sta,{tags} {cols} {ts + i}' + tags = self.genTags(index) + sql = f'{self.stable},{tags} {cols} {ts + i}' self.insert(sql, i) # run + def run(self): # seed #random.seed(int(time.time())) - self.dbname = "schema_change" + self.dbname = "eco_system" + self.stable = "sml_stb" # switch db tdSql.execute(f"use {self.dbname};") - tdSql.execute(f"drop table if exists sta;") + tdSql.execute(f"drop table if exists {self.stable};") # change meters try: - self.schemaless_insert(1000) + self.schemaless_insert(1000000) except: traceback.print_exc() From 8183be4afff416a66d9ec07879ac5e620d1e43d9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 May 2023 16:13:52 +0800 Subject: [PATCH 059/238] fix:move sleep logic from rpc thread to tmq thread & fix some error --- source/client/src/clientTmq.c | 9 ++++++--- source/dnode/vnode/src/tq/tq.c | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c08fbd0adf..b57ecfa845 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1275,7 +1275,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { // in case of consumer mismatch, wait for 500ms and retry if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { - taosMsleep(500); +// taosMsleep(500); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, wait for 500ms and set status to be RECOVER", tmq->consumerId); @@ -1289,8 +1289,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; taosWriteQitem(tmq->mqueue, pRspWrapper); - } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert - taosMsleep(500); +// } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert +// taosMsleep(5); } else{ tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, vgId, epoch, tstrerror(code), requestId); @@ -1731,6 +1731,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p // broadcast the poll request to all related vnodes static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { + if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){ + return 0; + } int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ce52277d71..91524f7a95 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -360,22 +360,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { // 2. check re-balance status if (pHandle->consumerId != consumerId) { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_INVALID_MSG; + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; taosWUnLockLatch(&pTq->lock); return -1; } // 3. update the epoch value - int32_t savedEpoch = pHandle->epoch; - if (savedEpoch <= reqEpoch) { - tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, - reqEpoch); - pHandle->epoch = reqEpoch; - }else { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", - consumerId, TD_VID(pTq->pVnode), req.subKey, savedEpoch, reqEpoch); + if (pHandle->epoch > reqEpoch) { + tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->epoch, reqEpoch); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; taosWUnLockLatch(&pTq->lock); return -1; @@ -395,6 +390,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } + if (pHandle->epoch < reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, pHandle->epoch, reqEpoch); + pHandle->epoch = reqEpoch; + } + char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, @@ -577,7 +577,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); } - atomic_add_fetch_32(&pHandle->epoch, 1); +// atomic_add_fetch_32(&pHandle->epoch, 1); // kill executing task if(tqIsHandleExec(pHandle)) { From 7b735c67d0a84f001c41c8f1145655cd3ddb40b4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 May 2023 16:36:25 +0800 Subject: [PATCH 060/238] fix:move sleep logic from rpc thread to tmq thread & fix some error --- source/dnode/vnode/src/tq/tq.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 91524f7a95..db03e97556 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -367,15 +367,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } - // 3. update the epoch value - if (pHandle->epoch > reqEpoch) { - tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", - consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->epoch, reqEpoch); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - taosWUnLockLatch(&pTq->lock); - return -1; - } - bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); @@ -390,6 +381,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } + // 3. update the epoch value if (pHandle->epoch < reqEpoch) { tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, pHandle->epoch, reqEpoch); pHandle->epoch = reqEpoch; From c9add944209c4e314ce54bf0f8975e4f8bdc7784 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 May 2023 09:58:52 +0800 Subject: [PATCH 061/238] docs: update readme main (#21401) * docs: update readme with libgflags * docs: update README-CN.md --- README-CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-CN.md b/README-CN.md index a6dfefc47a..cee6f97e9a 100644 --- a/README-CN.md +++ b/README-CN.md @@ -15,7 +15,7 @@ [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -简体中文 | [English](README.md) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/) +简体中文 | [English](README.md) | [TDengine 云服务](https://cloud.taosdata.com/?utm_medium=cn&utm_source=github) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/) # TDengine 简介 From 28d562a55d14d8e7dfb6b19e3081937fab3c239a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 22 May 2023 03:56:11 +0000 Subject: [PATCH 062/238] add parameter --- docs/en/14-reference/12-config/index.md | 19 +++++++++++++++++++ docs/zh/14-reference/12-config/index.md | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 430487a3af..cb9bebd723 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -97,6 +97,25 @@ The parameters described in this document by the effect that they have on the sy | Value Range | 10-50000000 | | Default Value | 5000 | +### numOfRpcSessions + +| Attribute | Description | +| ------------- | ---------------------------------------------------- | +| Applicable | Client/Server | +| Meaning | The maximum number of connection to create | +| Value Range | 100-100000 | +| Default Value | 10000 | + +### timeToGetAvailableConn + +| Attribute | Description | +| ------------- | ---------------------------------------------------- | +| Applicable | Client/Server | +| Meaning | The maximum waiting time to get avaliable conn | +| Value Range | 10-50000000(ms) | +| Default Value | 500000 | + + ## Monitoring Parameters :::note diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index e5efd77f80..9533f82ed8 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -97,6 +97,25 @@ taos --dump-config | 取值范围 | 10-50000000 | | 缺省值 | 5000 | +### numOfRpcSessions + +| 属性 | 说明 | +| -------- | ----------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 一个客户端能创建的最大连接数 | +| 取值范围 | 100-100000 | +| 缺省值 | 10000 | + +### timeToGetAvailableConn + +| 属性 | 说明 | +| -------- | ----------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 |获得可用连接的最长等待时间 | +| 取值范围 | 10-50000000(单位为毫秒) | +| 缺省值 | 500000 | + + ## 监控相关 :::note From 591b51f624139db99825be975199c25c7d094c2e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 May 2023 12:42:21 +0800 Subject: [PATCH 063/238] fix:set task code success after kill --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index db03e97556..f2956ea488 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -575,7 +575,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg if(tqIsHandleExec(pHandle)) { qTaskInfo_t pTaskInfo = pHandle->execHandle.task; if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); } // if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { From f67caaed1f5359ab806933603d40c41729e7ee24 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 22 May 2023 06:09:55 +0000 Subject: [PATCH 064/238] opt compile --- docs/zh/14-reference/12-config/index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 9533f82ed8..90070098b8 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -91,20 +91,20 @@ taos --dump-config ### maxShellConns | 属性 | 说明 | -| -------- | ----------------------- | +| --------| ----------------------- | | 适用范围 | 仅服务端适用 | -| 含义 | 一个 dnode 容许的连接数 | +| 含义 | 一个 dnode 容许的连接数 | | 取值范围 | 10-50000000 | -| 缺省值 | 5000 | +| 缺省值 | 5000 | ### numOfRpcSessions -| 属性 | 说明 | -| -------- | ----------------------- | -| 适用范围 | 客户端和服务端都适用 | -| 含义 | 一个客户端能创建的最大连接数 | +| 属性 | 说明 | +| --------| ---------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 一个客户端能创建的最大连接数| | 取值范围 | 100-100000 | -| 缺省值 | 10000 | +| 缺省值 | 10000 | ### timeToGetAvailableConn From d6b188f15d36a506a95ce3bf663700da9a61bc0c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 22 May 2023 06:10:00 +0000 Subject: [PATCH 065/238] opt compile --- docs/zh/14-reference/12-config/index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 90070098b8..ed6c9b4505 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -99,21 +99,21 @@ taos --dump-config ### numOfRpcSessions -| 属性 | 说明 | +| 属性 | 说明 | | --------| ---------------------- | | 适用范围 | 客户端和服务端都适用 | | 含义 | 一个客户端能创建的最大连接数| | 取值范围 | 100-100000 | -| 缺省值 | 10000 | +| 缺省值 | 10000 | ### timeToGetAvailableConn -| 属性 | 说明 | -| -------- | ----------------------- | -| 适用范围 | 客户端和服务端都适用 | -| 含义 |获得可用连接的最长等待时间 | -| 取值范围 | 10-50000000(单位为毫秒) | -| 缺省值 | 500000 | +| 属性 | 说明 | +| -------- | --------------------| +| 适用范围 | 客户端和服务端都适用 | +| 含义 |获得可用连接的最长等待时间| +| 取值范围 | 10-50000000(单位为毫秒)| +| 缺省值 | 500000 | ## 监控相关 From ae515851c4d753162f5bcd10aa940658e377055f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 May 2023 15:32:30 +0800 Subject: [PATCH 066/238] fix:remove get schema from previous for taosx --- source/dnode/vnode/src/meta/metaQuery.c | 38 ------------------------- 1 file changed, 38 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d464f64de3..428d201a17 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -387,42 +387,6 @@ _query: tDecoderClear(&dc); goto _exit; } - { // Traverse to find the previous qualified data - TBC *pCur; - tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); - STbDbKey key = {.version = sver, .uid = INT64_MAX}; - int c = 0; - tdbTbcMoveTo(pCur, &key, sizeof(key), &c); - if (c < 0) { - tdbTbcMoveToPrev(pCur); - } - - void *pKey = NULL; - void *pVal = NULL; - int vLen = 0, kLen = 0; - while (1) { - int32_t ret = tdbTbcPrev(pCur, &pKey, &kLen, &pVal, &vLen); - if (ret < 0) break; - - STbDbKey *tmp = (STbDbKey *)pKey; - if (tmp->uid != uid) { - continue; - } - SDecoder dcNew = {0}; - SMetaEntry meNew = {0}; - tDecoderInit(&dcNew, pVal, vLen); - metaDecodeEntry(&dcNew, &meNew); - pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow); - tDecoderClear(&dcNew); - tdbTbcClose(pCur); - tdbFree(pKey); - tdbFree(pVal); - goto _exit; - } - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCur); - } } else if (me.type == TSDB_CHILD_TABLE) { uid = me.ctbEntry.suid; tDecoderClear(&dc); @@ -447,7 +411,6 @@ _query: tDecoderClear(&dc); _exit: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } @@ -455,7 +418,6 @@ _exit: return pSchema; _err: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } From 7b20d2fa7d63af8959acaf29892a892aece67b71 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 22 May 2023 20:10:13 +0800 Subject: [PATCH 067/238] update crash_gen --- tests/pytest/crash_gen/crash_gen_main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index ec588659e9..9228cecdb5 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1722,12 +1722,14 @@ class TaskCreateDb(StateTransitionTask): vg_nums = random.randint(1, 8) cache_model = Dice.choice(['none', 'last_row', 'last_value', 'both']) buffer = random.randint(3, 128) + walRetentionPeriod = random.randint(1, 10000) dbName = self._db.getName() - self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} ".format(dbName, repStr, + self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} wal_retention_period {} ".format(dbName, repStr, updatePostfix, vg_nums, cache_model, - buffer)) + buffer, + walRetentionPeriod)) if dbName == "db_0" and Config.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix)) From d8387b95ff13682b3706bd9fbb06cf8094596b32 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 May 2023 20:14:43 +0800 Subject: [PATCH 068/238] fix:set task status to 0 in inti --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/executor/src/executor.c | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index f5f7218ab9..fe949d92ce 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,7 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); -//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f2956ea488..85d8d073f7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -370,7 +370,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); -// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); + qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a73deffa52..eea542e042 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,10 +180,10 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } -//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { -// SExecTaskInfo* pTaskInfo = tinfo; -// pTaskInfo->code = code; -//} +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { + SExecTaskInfo* pTaskInfo = tinfo; + pTaskInfo->code = code; +} int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { From ca479f57dfd40dc57503a33065f1798c7ff875fb Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Mon, 22 May 2023 21:40:38 +0800 Subject: [PATCH 069/238] Remove taosx, connector, explorer and taoskeeper in cloud tdengine --- packaging/docker/dockerbuild.sh | 10 ++++++++++ packaging/tools/makeclient.sh | 4 ++-- packaging/tools/makepkg.sh | 31 ++++++++++++++++++------------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 4b6fc8576b..ee09854176 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -123,6 +123,16 @@ else echo "Unknown cpuType: ${cpuType}" exit 1 fi +# check the tdengine cloud base image existed or not +if [ "$cloudBuild" == "y" ]; then + CloudBase=$(docker images | grep tdengine/tdengine-cloud-base ||:) + if [[ "$CloudBase" == "" ]]; then + echo "Rebuild tdengine cloud base image..." + docker build --rm -f "${communityDir}/packaging/docker/DockerfileCloud.base" -t tdengine/tdengine-cloud-base "." --build-arg cpuType=${cpuTypeAlias} + else + echo "Already found tdengine cloud base image" + fi +fi docker build --rm -f "${Dockerfile}" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias} if [ "$cloudBuild" != "y" ]; then diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 28dc770755..c4b074f903 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -175,7 +175,7 @@ if [ "$pagMode" == "lite" ]; then fi chmod a+x ${install_dir}/install_client.sh -if [[ $productName == "TDengine" ]]; then +if [[ $productName == "TDengine" ]] && [ "$verMode" != "cloud" ]; then # Copy example code mkdir -p ${install_dir}/examples examples_dir="${top_dir}/examples" @@ -191,7 +191,7 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json fi - if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then + if [ "$verMode" == "cluster" ]; then # Copy connector connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 3da005c405..7bf81d2807 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -69,25 +69,30 @@ if [ "$pagMode" == "lite" ]; then bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark " taostools_bin_files="" else - - wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ + if [ "$verMode" == "cloud" ]; then + taostools_bin_files=" ${build_dir}/bin/taosdump \ + ${build_dir}/bin/taosBenchmark" + else + wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ || echo "failed to download TDinsight.sh" - # download TDinsight caches - orig_pwd=$(pwd) - tdinsight_caches="" - cd ${build_dir}/bin/ && \ - chmod +x TDinsight.sh - ./TDinsight.sh --download-only ||: -# tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ") - cd $orig_pwd - echo "TDinsight caches: $tdinsight_caches" + # download TDinsight caches + orig_pwd=$(pwd) + tdinsight_caches="" + cd ${build_dir}/bin/ && \ + chmod +x TDinsight.sh + ./TDinsight.sh --download-only ||: + # tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ") + cd $orig_pwd + echo "TDinsight caches: $tdinsight_caches" - taostools_bin_files=" ${build_dir}/bin/taosdump \ + taostools_bin_files=" ${build_dir}/bin/taosdump \ ${build_dir}/bin/taosBenchmark \ ${build_dir}/bin/TDinsight.sh \ ${build_dir}/bin/tdengine-datasource.zip \ ${build_dir}/bin/tdengine-datasource.zip.md5sum" + fi + [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer') @@ -334,7 +339,7 @@ mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : # Copy connector -if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then +if [ "$verMode" == "cluster" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then From ed612088c05e40880520a1396f630131d51bd49a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 May 2023 10:32:13 +0800 Subject: [PATCH 070/238] fix:lost data because tsdbreader or task is killed --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 14 +++++++------- source/libs/executor/src/executor.c | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index fe949d92ce..f5f7218ab9 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,7 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 85d8d073f7..17eac7d096 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -370,7 +370,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); - qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); +// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; @@ -572,16 +572,16 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // atomic_add_fetch_32(&pHandle->epoch, 1); // kill executing task - if(tqIsHandleExec(pHandle)) { - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } +// if(tqIsHandleExec(pHandle)) { +// qTaskInfo_t pTaskInfo = pHandle->execHandle.task; +// if (pTaskInfo != NULL) { +// qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); +// } // if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { // qStreamCloseTsdbReader(pTaskInfo); // } - } +// } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); taosWUnLockLatch(&pTq->lock); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index eea542e042..a73deffa52 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,10 +180,10 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { - SExecTaskInfo* pTaskInfo = tinfo; - pTaskInfo->code = code; -} +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { +// SExecTaskInfo* pTaskInfo = tinfo; +// pTaskInfo->code = code; +//} int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { From 00524b3185f6eac4cec7148ccc4fd98d7cce5341 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Tue, 23 May 2023 11:36:42 +0800 Subject: [PATCH 071/238] remove the taosdump in cloud --- packaging/tools/makepkg.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 7bf81d2807..fc02a288ca 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -70,8 +70,7 @@ if [ "$pagMode" == "lite" ]; then taostools_bin_files="" else if [ "$verMode" == "cloud" ]; then - taostools_bin_files=" ${build_dir}/bin/taosdump \ - ${build_dir}/bin/taosBenchmark" + taostools_bin_files="${build_dir}/bin/taosBenchmark" else wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ From 13a6eb177c3c9682d2c269e0143d4065e4b732a6 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Tue, 23 May 2023 11:38:04 +0800 Subject: [PATCH 072/238] add space before the bin files --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index fc02a288ca..748b79bd7f 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -70,7 +70,7 @@ if [ "$pagMode" == "lite" ]; then taostools_bin_files="" else if [ "$verMode" == "cloud" ]; then - taostools_bin_files="${build_dir}/bin/taosBenchmark" + taostools_bin_files=" ${build_dir}/bin/taosBenchmark" else wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ From 8c1c2a2f976fd49657ea1ff7beb5642e3c86a1bb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 May 2023 17:47:17 +0800 Subject: [PATCH 073/238] fix:add grammar support --- source/libs/parser/inc/sql.y | 4 ++-- source/libs/parser/src/parAstCreater.c | 27 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 5918b488b4..4a457838d4 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -530,9 +530,9 @@ cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C). cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) WITH META AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, true); } cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, false); } + AS STABLE full_table_name(C) where_clause_opt(D). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, false, D); } cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - WITH META AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, true); } + WITH META AS STABLE full_table_name(C) where_clause_opt(D). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, true, D); } cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); } cmd ::= DROP CONSUMER GROUP exists_opt(A) cgroup_name(B) ON topic_name(C). { pCxt->pRootNode = createDropCGroupStmt(pCxt, A, &B, &C); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5a47ed731d..7258f6c415 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1713,6 +1713,33 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); pStmt->ignoreExists = ignoreExists; pStmt->withMeta = withMeta; + +// SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); +// if (NULL == pSelect) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// sprintf(pSelect->stmtName, "%p", pSelect); +// +// SRealTableNode* pRealTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); +// if (NULL == pRealTable) { +// nodesDestroyNode((SNode*)pSelect); +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb); +// snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); +// snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); +// pSelect->pFromTable = (SNode*)pRealTable; +// +// if (numOfProjs >= 0) { +// pSelect->pProjectionList = createProjectCols(numOfProjs, pProjCol); +// if (NULL == pSelect->pProjectionList) { +// nodesDestroyNode((SNode*)pSelect); +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// } +// +// pStmt->pQuery = pSelect; + strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName); strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); nodesDestroyNode(pRealTable); From a54eaa20ffe37bcf6ffd59291ed0b52c530f78b5 Mon Sep 17 00:00:00 2001 From: xleili Date: Tue, 23 May 2023 18:11:00 +0800 Subject: [PATCH 074/238] build: release ver-3.0.4.2 --- cmake/cmake.version | 2 +- packaging/tools/makepkg.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 3166a0695c..1d43986de4 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.4.1") + SET(TD_VER_NUMBER "3.0.4.2") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 748b79bd7f..e297902bf9 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -345,7 +345,7 @@ if [ "$verMode" == "cluster" ]; then tmp_pwd=`pwd` cd ${install_dir}/connector if [ ! -d taos-connector-jdbc ];then - git clone -b 3.1.0 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + git clone -b 3.2.1 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: fi cd taos-connector-jdbc mvn clean package -Dmaven.test.skip=true From 34cf6aad3014516cb6bd69ea4797bc5a08295085 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 23 May 2023 18:29:53 +0800 Subject: [PATCH 075/238] fix: alter dnode option has no effect --- source/dnode/mnode/impl/src/mndDnode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 1f58ae97a3..e5ee4b8a21 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -965,7 +965,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); mInfo("dnode:%d, send config req to dnode, app:%p config:%s value:%s", cfgReq.dnodeId, pReq->info.ahandle, dcfgReq.config, dcfgReq.value); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .info = pReq->info}; + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; tmsgSendReq(&epSet, &rpcMsg); code = 0; } From 5bedf6b19ae337f77a43941ae6bc58e153d26517 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 23 May 2023 19:21:52 +0800 Subject: [PATCH 076/238] enh: support config batch rows number when import data from csv file --- include/common/tglobal.h | 2 +- source/common/src/tglobal.c | 10 +++---- source/libs/parser/src/parInsertSql.c | 4 ++- source/libs/parser/src/parInsertUtil.c | 37 +++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 6fde7b48a2..44276102cf 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -122,7 +122,7 @@ extern bool tsUseAdapter; // client extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; -extern int32_t tsMaxMemUsedByInsert; +extern int32_t tsMaxInsertBatchRows; // build info extern char version[]; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index b85dfa80b0..bef63d8a49 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -148,8 +148,8 @@ int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 database precision unit for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; -// maximum memory allowed to be allocated for a single csv load (in MB) -int32_t tsMaxMemUsedByInsert = 1024; +// maximum batch rows numbers imported from a single csv load +int32_t tsMaxInsertBatchRows = 1000000; float tsSelectivityRatio = 1.0; int32_t tsTagFilterResCacheSize = 1024 * 10; @@ -340,7 +340,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; - if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; + if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; @@ -725,7 +725,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { // tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; // tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; @@ -987,7 +987,7 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; } else if (strcasecmp("maxRetryWaitTime", name) == 0) { tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 02de9f227d..6751b34d8e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1552,7 +1552,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxInsertBatchRows) { pStmt->fileProcessing = true; break; } @@ -1561,6 +1561,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } taosMemoryFree(pLine); + parserDebug("0x%" PRIx64 " %d rows have been parsed", pCxt->pComCxt->requestId, *pNumOfRows); + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && !pStmt->fileProcessing) { code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index f921094752..952a927136 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -272,6 +272,41 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat return code; } +static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { + int32_t code = TSDB_CODE_SUCCESS; + SSubmitTbData* pTmp = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pTmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTmp->flags = pSrc->flags; + pTmp->suid = pSrc->suid; + pTmp->uid = pSrc->uid; + pTmp->sver = pSrc->sver; + pTmp->pCreateTbReq = pSrc->pCreateTbReq; + if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + pTmp->aCol = taosArrayInit(128, sizeof(SColData)); + if (NULL == pTmp->aCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } else { + pTmp->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pTmp->aRowP) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } + } + + taosMemoryFree(pSrc); + if (TSDB_CODE_SUCCESS == code) { + *pDst = pTmp; + } + + return code; +} + + static void resetColValues(SArray* pValues) { int32_t num = taosArrayGetSize(pValues); for (int32_t i = 0; i < num; ++i) { @@ -381,7 +416,7 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx } } taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); - taosMemoryFreeClear(pTableCxt->pData); + rebuildTableData(pTableCxt->pData, &pTableCxt->pData); qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); From b52f867a1f933772765b44d6b9a4297cc489a7fc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 24 May 2023 09:56:59 +0800 Subject: [PATCH 077/238] fix: rebuild table data issue --- source/libs/parser/src/parInsertUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 952a927136..de7d154db6 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -282,7 +282,7 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { pTmp->suid = pSrc->suid; pTmp->uid = pSrc->uid; pTmp->sver = pSrc->sver; - pTmp->pCreateTbReq = pSrc->pCreateTbReq; + pTmp->pCreateTbReq = NULL; if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { pTmp->aCol = taosArrayInit(128, sizeof(SColData)); if (NULL == pTmp->aCol) { From 9a421a92f90aefb83c54f870fe18c672a3bd904e Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 24 May 2023 10:32:10 +0800 Subject: [PATCH 078/238] fix: core dump when select with interval caused by order --- source/libs/executor/src/filloperator.c | 2 +- source/libs/executor/src/tfill.c | 2 +- tests/script/tsim/query/interval.sim | 36 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 73222ee375..6d4b8570d1 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -258,7 +258,7 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t STimeWindow w = {0}; int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - getInitialStartTimeWindow(pInterval, startKey, &w, order); + getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index c98746c241..a39521f25c 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -561,7 +561,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma ASSERT(numOfRes >= numOfRows); } else { // reach the end of data if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || - (ekey1 >= pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { + (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 833da4a8ba..e2b0d219cb 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -170,6 +170,42 @@ endi # return -1 #endi +print ================ step10 +print -------- create database and table +sql create database if not exists test +sql use test +sql create stable st (ts timestamp, c2 int) tags(tg int) +print -------- insert 300 rows data +$i = 0 +while $i < 300 + $t = 1577807983000 + $cc = $i * 1000 + $t = $t + $cc + sql select $i % 3 + if $data00 != 0.000000000 then + goto continue_while + endi + sql select $i % 4 + if $data00 == 0.000000000 then + goto continue_while + endi + sql insert into t1 using st tags(1) values ( $t , $i ) + continue_while: + $i = $i + 1 +endw + +$ms1 = 1577808120000 +$ms2 = 1577808000000 +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts asc +print ----> select asc rows: $rows +$asc_rows = $rows +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts desc +print ----> select desc rows: $rows +$desc_rows = $rows +if $desc_rows != $asc_rows then + return -1 +endi + print =============== clear #sql drop database $db #sql select * from information_schema.ins_databases From d6d0bf0e41ef5a0490376d2ce8658826868b06f9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 May 2023 10:47:26 +0800 Subject: [PATCH 079/238] enh: add varchar sma --- source/common/src/tdataformat.c | 52 ++++++++++++++++--- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index b18bd882ae..55204045ba 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2441,7 +2441,7 @@ _exit: int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, char *data) { int32_t code = 0; - if(data == NULL){ + if (data == NULL) { for (int32_t i = 0; i < nRows; ++i) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); } @@ -2455,8 +2455,9 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; } else { - if(ASSERT(varDataTLen(data + offset) <= bytes)){ - uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes); + if (ASSERT(varDataTLen(data + offset) <= bytes)) { + uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), + bytes); code = TSDB_CODE_INVALID_PARA; goto _exit; } @@ -2508,7 +2509,7 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { ASSERT(pColData->type == pBind->buffer_type); } - + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { @@ -3521,6 +3522,43 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum } } +static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = 0; + *numOfNull = 0; + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + case (HAS_NONE | HAS_NULL): + *numOfNull = pColData->nVal; + break; + case HAS_VALUE: + *numOfNull = 0; + break; + case (HAS_VALUE | HAS_NULL): + case (HAS_VALUE | HAS_NONE): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT1(pColData->pBitMap, iVal) == 0) { + (*numOfNull)++; + } + } + break; + case (HAS_VALUE | HAS_NONE | HAS_NULL): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT2(pColData->pBitMap, iVal) != 2) { + (*numOfNull)++; + } + } + break; + default: + ASSERT(0); + break; + } +} + void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = { NULL, tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL @@ -3530,14 +3568,14 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_ tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARCHAR tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_NCHAR tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON NULL, // TSDB_DATA_TYPE_VARBINARY NULL, // TSDB_DATA_TYPE_DECIMAL NULL, // TSDB_DATA_TYPE_BLOB diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 50fd9d7aa7..b25ab393da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -523,7 +523,7 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type) || ((pColData->flag & HAS_VALUE) == 0)) continue; + if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue; SColumnDataAgg sma = {.colId = pColData->cid}; tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); From 7359719429b5403bbfd3c28a645fad72b534b356 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Wed, 24 May 2023 13:57:59 +0800 Subject: [PATCH 080/238] implement TD-24378 --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 748b79bd7f..9c141a4f37 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -428,7 +428,7 @@ if [ "$exitcode" != "0" ]; then exit $exitcode fi -if [ -n "${taostools_bin_files}" ]; then +if [ -n "${taostools_bin_files}" ] && [ "$verMode" != "cloud" ]; then wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh" if [ "$osType" != "Darwin" ]; then tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || : From 89bea7100f5cf92f7aa332b167c6d6a850ddd539 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 24 May 2023 00:31:05 -0700 Subject: [PATCH 081/238] fix: fix count var type error --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 +++++++++++--- source/libs/executor/src/scanoperator.c | 5 +++-- source/libs/function/src/builtinsimpl.c | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cb83a561d2..5a7bebebff 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -184,7 +184,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pT SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); void tsdbReaderClose(STsdbReader *pReader); int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4acc784aee..5d970ce6c3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4973,7 +4973,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { return code; } -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { +static bool doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { + bool hasNullSMA = false; // do fill all null column value SMA info int32_t i = 0, j = 0; int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); @@ -4993,6 +4994,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; size++; + hasNullSMA = true; } j += 1; } @@ -5003,12 +5005,15 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; + hasNullSMA = true; } j++; } + + return hasNullSMA; } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool *hasNullSMA) { SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; int32_t code = 0; @@ -5072,7 +5077,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } // do fill all null column value SMA info - doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg); + if (doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg)) { + *hasNullSMA = true; + return TSDB_CODE_SUCCESS; + } size_t size = taosArrayGetSize(pSup->pColAgg); int32_t i = 0, j = 0; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a001d99408..c1430c4ce0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -228,12 +228,13 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { bool allColumnsHaveAgg = true; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + bool hasNullSMA = false; + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg, &hasNullSMA); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - if (!allColumnsHaveAgg) { + if (!allColumnsHaveAgg || hasNullSMA) { return false; } return true; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4c019b3e71..cdecc8b8cc 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -500,7 +500,7 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) { */ SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) { + if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; } else { if (pInputCol->hasNull) { From bd45785dfdd4e88b815069eabd8c6eeebd6f3c78 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 24 May 2023 01:30:48 -0700 Subject: [PATCH 082/238] add test cases --- tests/system-test/2-query/count_null.py | 144 ++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 tests/system-test/2-query/count_null.py diff --git a/tests/system-test/2-query/count_null.py b/tests/system-test/2-query/count_null.py new file mode 100644 index 0000000000..6d2c8db8d6 --- /dev/null +++ b/tests/system-test/2-query/count_null.py @@ -0,0 +1,144 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def check_results(self): + tdSql.query(f"select count(*) from tb1") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c2) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c3) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c4) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c5) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c6) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c7) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb1") + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select count(*) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c2) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c3) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c4) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c5) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c6) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c7) from tb2") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb2") + tdSql.checkData(0, 0, 0) + + for i in range (3, 6): + tdSql.query(f"select count(*) from tb{i}") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c2) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c3) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c4) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c5) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c6) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c7) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c8) from tb{i}") + tdSql.checkData(0, 0, 10000) + + + def run(self): + dbname = 'db' + tbnames = ['tb1', 'tb2', 'tb3', 'tb4', 'tb5', 'tb6'] + num_rows = 20000 + num_tables = 6 + ts_base = 1685548800000 + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + for i in range (num_tables): + tdSql.execute( + f'''create table if not exists {dbname}.{tbnames[i]} + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) + + ''' + ) + + + tdLog.printNoPrefix("==========step2:insert data") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[0]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[1]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, null, null, null, null, null, null, null, 'binary', 'nchar')") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i < num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i >= num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + + tdLog.printNoPrefix("==========step3:check result in memory") + self.check_results(); + + tdLog.printNoPrefix("==========step3:check result from disk") + tdSql.execute(f"flush database db") + self.check_results(); + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From e4a3d1dfcf188ed8ceecfa4745df4c5bb2ade3af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 May 2023 18:26:02 +0800 Subject: [PATCH 083/238] fix: link rocksdb shared library --- cmake/cmake.options | 6 ++++++ cmake/rocksdb_CMakeLists.txt.in | 35 ++++++++++++++++++++++++--------- contrib/CMakeLists.txt | 2 ++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index 3bc5e202a5..555b72cbdf 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -77,6 +77,12 @@ ELSEIF (TD_DARWIN_64) ENDIF () ENDIF () +option( + BUILD_GEOS + "If build geos on Windows" + ON + ) + option( BUILD_SHARED_LIBS "" diff --git a/cmake/rocksdb_CMakeLists.txt.in b/cmake/rocksdb_CMakeLists.txt.in index ba4a404af6..e100b3a932 100644 --- a/cmake/rocksdb_CMakeLists.txt.in +++ b/cmake/rocksdb_CMakeLists.txt.in @@ -1,11 +1,28 @@ # rocksdb -ExternalProject_Add(rocksdb - GIT_REPOSITORY https://github.com/facebook/rocksdb.git - GIT_TAG v8.1.1 - SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - ) + +find_library(ROCKSDB_LIB NAMES rocksdb) +find_path(ROCKSDB_INCLUDE_DIR NAMES rocksdb/db.h) + +if (ROCKSDB_LIB AND ROCKSDB_INCLUDE_DIR) + set(ROCKSDB_FOUND TRUE) +endif() + +if(ROCKSDB_FOUND) + message(STATUS "Found rocksdb") +else() + message(STATUS "Building rocksdb from source") + + ExternalProject_Add(rocksdb + GIT_REPOSITORY https://github.com/facebook/rocksdb.git + GIT_TAG v8.1.1 + SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" + BINARY_DIR "${TD_CONTRIB_DIR}/rocksdb" + CONFIGURE_COMMAND "" + BUILD_COMMAND COMMAND $(MAKE) install-shared -j8 + INSTALL_COMMAND "" + TEST_COMMAND "" + ) +endif() + + diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b3333dee99..e3a0f72959 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -226,6 +226,7 @@ if(${BUILD_WITH_LEVELDB}) endif(${BUILD_WITH_LEVELDB}) # rocksdb +if (FALSE) # To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev if(${BUILD_WITH_ROCKSDB}) if(${TD_LINUX}) @@ -276,6 +277,7 @@ if(${BUILD_WITH_ROCKSDB}) PUBLIC $ ) endif(${BUILD_WITH_ROCKSDB}) +endif() # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev From 8537e4e80e0bb24a11cc911a2d36a53e4438ef15 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 24 May 2023 18:26:55 +0800 Subject: [PATCH 084/238] feat:add tag filter for stable subscribe --- include/common/tmsg.h | 3 ++ include/libs/executor/executor.h | 2 ++ source/common/src/tmsg.c | 15 ++++++++++ source/dnode/mnode/impl/src/mndScheduler.c | 2 ++ source/dnode/mnode/impl/src/mndTopic.c | 2 ++ source/dnode/vnode/src/inc/tq.h | 2 ++ source/dnode/vnode/src/tq/tq.c | 27 +++++++++++------ source/dnode/vnode/src/tq/tqMeta.c | 20 +++++++++---- source/dnode/vnode/src/tq/tqRead.c | 35 +++++----------------- source/libs/executor/src/executil.c | 16 +++++++++- source/libs/parser/src/parTranslater.c | 1 + 11 files changed, 82 insertions(+), 43 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1ea9714bf9..8fd09afb90 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1949,6 +1949,7 @@ typedef struct { char* ast; char subStbName[TSDB_TABLE_FNAME_LEN]; }; + char* subStbFilterAst; } SCMCreateTopicReq; int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); @@ -2758,6 +2759,7 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR tlen += taosEncodeString(buf, pReq->qmsg); } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { tlen += taosEncodeFixedI64(buf, pReq->suid); + tlen += taosEncodeString(buf, pReq->qmsg); } return tlen; } @@ -2773,6 +2775,7 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { buf = taosDecodeString(buf, &pReq->qmsg); } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { + buf = taosDecodeString(buf, &pReq->qmsg); buf = taosDecodeFixedI64(buf, &pReq->suid); } return (void*)buf; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index b7e6c42e3b..e3a75ecabc 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -82,6 +82,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); +int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* pTagCond, void* pTagIndexCond, SArray **tableList); + /** * set the task Id, usually used by message queue process * @param tinfo diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7102e556cc..1eb73ab842 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3841,6 +3841,10 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeI32(&encoder, strlen(pReq->sql)) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tEncodeI32(&encoder, strlen(pReq->subStbFilterAst)) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->subStbFilterAst) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3879,6 +3883,15 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tDecodeI32(&decoder, &astLen) < 0) return -1; + if (astLen > 0) { + pReq->subStbFilterAst = taosMemoryCalloc(1, astLen + 1); + if (pReq->subStbFilterAst == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->subStbFilterAst) < 0) return -1; + } + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -3889,6 +3902,8 @@ void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { taosMemoryFreeClear(pReq->sql); if (TOPIC_SUB_TYPE__COLUMN == pReq->subType) { taosMemoryFreeClear(pReq->ast); + }else if(TOPIC_SUB_TYPE__TABLE == pReq->subType) { + taosMemoryFreeClear(pReq->subStbFilterAst); } } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 734f624be0..0248a195db 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -591,6 +591,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + } else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ + pVgEp->qmsg = taosStrdup(pTopic->ast); } else { pVgEp->qmsg = taosStrdup(""); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index f6da370916..7d71aae3f4 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -462,6 +462,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.stbUid = pStb->uid; mndReleaseStb(pMnode, pStb); + topicObj.ast = taosStrdup(pCreate->ast); + topicObj.astLen = strlen(pCreate->ast) + 1; } /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/ /*topicObj.ast = NULL;*/ diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e431ca4a01..7895690f94 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -72,6 +72,8 @@ typedef struct { typedef struct { int64_t suid; + char* qmsg; // SubPlanToString + SNode* node; } STqExecTb; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 311d637be8..be3d6bc614 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -70,6 +70,8 @@ static void destroyTqHandle(void* data) { } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { walCloseReader(pData->pWalReader); tqCloseReader(pData->execHandle.pTqReader); + taosMemoryFreeClear(pData->execHandle.execTb.qmsg); + nodesDestroyNode(pData->execHandle.execTb.node); } if(pData->msg != NULL) { rpcFreeCont(pData->msg->pCont); @@ -470,7 +472,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle tqHandle = {0}; pHandle = &tqHandle; - uint64_t oldConsumerId = pHandle->consumerId; memcpy(pHandle->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN); pHandle->consumerId = req.newConsumerId; pHandle->epoch = -1; @@ -514,14 +515,22 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); pHandle->execHandle.execTb.suid = req.suid; + pHandle->execHandle.execTb.qmsg = req.qmsg; + req.qmsg = NULL; - SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); - vnodeGetCtbIdList(pVnode, req.suid, tbUidList); - tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pVnode->config.vgId, req.suid); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); + if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s", terrstr()); + return -1; } + + SArray* tbUidList = NULL; + ret = qGetTableList(req.suid, pVnode->pMeta, pVnode, pHandle->execHandle.execTb.node, NULL, &tbUidList); + if(ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, req.subKey, pHandle->consumerId); + taosArrayDestroy(tbUidList); + goto end; + } + tqDebug("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, req.suid); pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); @@ -532,8 +541,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); - tqDebug("try to persist handle %s consumer:0x%" PRIx64 " , old consumer:0x%" PRIx64, req.subKey, - pHandle->consumerId, oldConsumerId); + tqDebug("try to persist handle %s consumer:0x%" PRIx64, req.subKey, + pHandle->consumerId); ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index f3ecaa08f6..ee7af5b2bf 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -37,6 +37,7 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tEncodeI64(pEncoder, pHandle->execHandle.execTb.suid) < 0) return -1; + if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1; } tEndEncode(pEncoder); return pEncoder->pos; @@ -64,6 +65,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tDecodeI64(pDecoder, &pHandle->execHandle.execTb.suid) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1; } tEndDecode(pDecoder); return 0; @@ -336,13 +338,19 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); - vnodeGetCtbIdList(pTq->pVnode, handle.execHandle.execTb.suid, tbUidList); - tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); + if (nodesStringToNode(handle.execHandle.execTb.qmsg, &handle.execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s", terrstr()); + return -1; } + + SArray* tbUidList = NULL; + int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, handle.execHandle.execTb.node, NULL, &tbUidList); + if(ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle.subKey, handle.consumerId); + taosArrayDestroy(tbUidList); + goto end; + } + tqDebug("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid); handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 1fbdb25528..38f5307384 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1040,34 +1040,15 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { - SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); - SMetaReader mr = {0}; - metaReaderInit(&mr, pTq->pVnode->pMeta, 0); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); ++i) { - uint64_t* id = (uint64_t*)taosArrayGet(tbUidList, i); - - int32_t code = metaGetTableEntryByUidCache(&mr, *id); - if (code != TSDB_CODE_SUCCESS) { - tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); - continue; - } - - tDecoderClear(&mr.coder); - if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pTqHandle->execHandle.execTb.suid) { - tqDebug("table uid %" PRId64 " does not add to tq handle", *id); - continue; - } - - tqDebug("table uid %" PRId64 " add to tq handle", *id); - taosArrayPush(qa, id); + SArray* list = NULL; + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, pTqHandle->execHandle.execTb.node, NULL, &list); + if(ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); + taosArrayDestroy(list); + return ret; } - - metaReaderClear(&mr); - if (taosArrayGetSize(qa) > 0) { - tqReaderAddTbUidList(pTqHandle->execHandle.pTqReader, qa); - } - - taosArrayDestroy(qa); + tqReaderSetTbUidList(pTqHandle->execHandle.pTqReader, list); + taosArrayDestroy(list); } else { tqReaderRemoveTbUidList(pTqHandle->execHandle.pTqReader, tbUidList); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c51dc39b5b..1fb35ae271 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -935,6 +935,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; SScalarParam output = {0}; + SArray* pUidTagList = NULL; tagFilterAssist ctx = {0}; ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); @@ -954,7 +955,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; // int64_t stt = taosGetTimestampUs(); - SArray* pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo)); + pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo)); copyExistedUids(pUidTagList, pUidList); FilterCondType condType = checkTagCond(pTagCond); @@ -1121,6 +1122,19 @@ _end: return code; } +int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* pTagCond, void* pTagIndexCond, SArray **tableList){ + SScanPhysiNode node = {0}; + node.suid = suid; + node.uid = suid; + node.tableType = TSDB_SUPER_TABLE; + STableListInfo* pTableListInfo = tableListCreate(); + int code = getTableList(metaHandle, pVnode, &node, pTagCond, pTagIndexCond, pTableListInfo, "qGetTableList"); + *tableList = pTableListInfo->pTableList; + pTableListInfo->pTableList = NULL; + tableListDestroy(pTableListInfo); + return code; +} + size_t getTableTagsBufLen(const SNodeList* pGroups) { size_t keyLen = 0; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f4c86d4849..f90fe59908 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5764,6 +5764,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); tNameGetFullDbName(&name, pReq->subDbName); tNameExtractFullName(&name, pReq->subStbName); + code = nodesNodeToString(pStmt->pQuery, false, &pReq->subStbFilterAst, NULL); } else if ('\0' != pStmt->subDbName[0]) { pReq->subType = TOPIC_SUB_TYPE__DB; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->subDbName, strlen(pStmt->subDbName)); From 4d526ed0cdf40b7a22f15508768f3eb33c5863fa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 May 2023 18:59:09 +0800 Subject: [PATCH 085/238] change more --- cmake/rocksdb_CMakeLists.txt.in | 36 +++++++++---------------------- contrib/CMakeLists.txt | 4 +--- source/dnode/vnode/CMakeLists.txt | 2 +- source/libs/stream/CMakeLists.txt | 2 +- 4 files changed, 13 insertions(+), 31 deletions(-) diff --git a/cmake/rocksdb_CMakeLists.txt.in b/cmake/rocksdb_CMakeLists.txt.in index e100b3a932..bb86cafb63 100644 --- a/cmake/rocksdb_CMakeLists.txt.in +++ b/cmake/rocksdb_CMakeLists.txt.in @@ -1,28 +1,12 @@ # rocksdb - -find_library(ROCKSDB_LIB NAMES rocksdb) -find_path(ROCKSDB_INCLUDE_DIR NAMES rocksdb/db.h) - -if (ROCKSDB_LIB AND ROCKSDB_INCLUDE_DIR) - set(ROCKSDB_FOUND TRUE) -endif() - -if(ROCKSDB_FOUND) - message(STATUS "Found rocksdb") -else() - message(STATUS "Building rocksdb from source") - - ExternalProject_Add(rocksdb - GIT_REPOSITORY https://github.com/facebook/rocksdb.git - GIT_TAG v8.1.1 - SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" - BINARY_DIR "${TD_CONTRIB_DIR}/rocksdb" - CONFIGURE_COMMAND "" - BUILD_COMMAND COMMAND $(MAKE) install-shared -j8 - INSTALL_COMMAND "" - TEST_COMMAND "" - ) -endif() - - +ExternalProject_Add(rocksdb + GIT_REPOSITORY https://github.com/facebook/rocksdb.git + GIT_TAG v8.1.1 + SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" + BINARY_DIR "${TD_CONTRIB_DIR}/rocksdb" + CONFIGURE_COMMAND "" + BUILD_COMMAND COMMAND $(MAKE) install-shared -j8 + INSTALL_COMMAND "" + TEST_COMMAND "" + ) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e3a0f72959..ec48a85c33 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -226,7 +226,6 @@ if(${BUILD_WITH_LEVELDB}) endif(${BUILD_WITH_LEVELDB}) # rocksdb -if (FALSE) # To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev if(${BUILD_WITH_ROCKSDB}) if(${TD_LINUX}) @@ -270,14 +269,13 @@ if(${BUILD_WITH_ROCKSDB}) option(WITH_TOOLS "" OFF) option(WITH_LIBURING "" OFF) - option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) + option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON) add_subdirectory(rocksdb EXCLUDE_FROM_ALL) target_include_directories( rocksdb PUBLIC $ ) endif(${BUILD_WITH_ROCKSDB}) -endif() # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e8660cd6ad..9b39516ab0 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -101,7 +101,7 @@ target_link_libraries( # PUBLIC bdb # PUBLIC scalar - PUBLIC rocksdb + PUBLIC rocksdb-shared PUBLIC transport PUBLIC stream PUBLIC index diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 2edbc44aae..acf563e822 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream - PUBLIC rocksdb tdb + PUBLIC rocksdb-shared tdb PRIVATE os util transport qcom executor wal ) From 357e86b9945c6a2765da47ddad6f6d82708a06ad Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 24 May 2023 18:59:47 +0800 Subject: [PATCH 086/238] feat:add tag filter for stable subscribe --- include/common/tmsg.h | 7 ++----- source/common/src/tmsg.c | 27 +++++++------------------- source/libs/parser/src/parTranslater.c | 4 +++- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8fd09afb90..a7de03fbce 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1945,11 +1945,8 @@ typedef struct { int8_t withMeta; char* sql; char subDbName[TSDB_DB_FNAME_LEN]; - union { - char* ast; - char subStbName[TSDB_TABLE_FNAME_LEN]; - }; - char* subStbFilterAst; + char* ast; + char subStbName[TSDB_TABLE_FNAME_LEN]; } SCMCreateTopicReq; int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 1eb73ab842..cb4d0c9034 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3832,19 +3832,16 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeI8(&encoder, pReq->withMeta) < 0) return -1; if (tEncodeCStr(&encoder, pReq->subDbName) < 0) return -1; if (TOPIC_SUB_TYPE__DB == pReq->subType) { - } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tEncodeCStr(&encoder, pReq->subStbName) < 0) return -1; } else { + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tEncodeCStr(&encoder, pReq->subStbName) < 0) return -1; + } if (tEncodeI32(&encoder, strlen(pReq->ast)) < 0) return -1; if (tEncodeCStr(&encoder, pReq->ast) < 0) return -1; } if (tEncodeI32(&encoder, strlen(pReq->sql)) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; - if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tEncodeI32(&encoder, strlen(pReq->subStbFilterAst)) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->subStbFilterAst) < 0) return -1; - } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3866,9 +3863,10 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeI8(&decoder, &pReq->withMeta) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->subDbName) < 0) return -1; if (TOPIC_SUB_TYPE__DB == pReq->subType) { - } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tDecodeCStrTo(&decoder, pReq->subStbName) < 0) return -1; } else { + if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { + if (tDecodeCStrTo(&decoder, pReq->subStbName) < 0) return -1; + } if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (astLen > 0) { pReq->ast = taosMemoryCalloc(1, astLen + 1); @@ -3883,15 +3881,6 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; } - if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { - if (tDecodeI32(&decoder, &astLen) < 0) return -1; - if (astLen > 0) { - pReq->subStbFilterAst = taosMemoryCalloc(1, astLen + 1); - if (pReq->subStbFilterAst == NULL) return -1; - if (tDecodeCStrTo(&decoder, pReq->subStbFilterAst) < 0) return -1; - } - } - tEndDecode(&decoder); tDecoderClear(&decoder); @@ -3900,10 +3889,8 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) { taosMemoryFreeClear(pReq->sql); - if (TOPIC_SUB_TYPE__COLUMN == pReq->subType) { + if (TOPIC_SUB_TYPE__DB != pReq->subType) { taosMemoryFreeClear(pReq->ast); - }else if(TOPIC_SUB_TYPE__TABLE == pReq->subType) { - taosMemoryFreeClear(pReq->subStbFilterAst); } } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f90fe59908..7c11443acb 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5764,7 +5764,9 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); tNameGetFullDbName(&name, pReq->subDbName); tNameExtractFullName(&name, pReq->subStbName); - code = nodesNodeToString(pStmt->pQuery, false, &pReq->subStbFilterAst, NULL); + if(pStmt->pQuery != NULL) { + code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); + } } else if ('\0' != pStmt->subDbName[0]) { pReq->subType = TOPIC_SUB_TYPE__DB; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->subDbName, strlen(pStmt->subDbName)); From b708a3c9dd35cabaf3d831dbb007f6d13d76c029 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 May 2023 18:59:54 +0800 Subject: [PATCH 087/238] more --- cmake/rocksdb_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/rocksdb_CMakeLists.txt.in b/cmake/rocksdb_CMakeLists.txt.in index bb86cafb63..86d3ae946a 100644 --- a/cmake/rocksdb_CMakeLists.txt.in +++ b/cmake/rocksdb_CMakeLists.txt.in @@ -6,7 +6,7 @@ ExternalProject_Add(rocksdb SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" BINARY_DIR "${TD_CONTRIB_DIR}/rocksdb" CONFIGURE_COMMAND "" - BUILD_COMMAND COMMAND $(MAKE) install-shared -j8 + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" ) From e60ca238342fb1fbb68f3c169817330e980001f8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 May 2023 19:01:04 +0800 Subject: [PATCH 088/238] more --- cmake/rocksdb_CMakeLists.txt.in | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/rocksdb_CMakeLists.txt.in b/cmake/rocksdb_CMakeLists.txt.in index 86d3ae946a..ba4a404af6 100644 --- a/cmake/rocksdb_CMakeLists.txt.in +++ b/cmake/rocksdb_CMakeLists.txt.in @@ -4,7 +4,6 @@ ExternalProject_Add(rocksdb GIT_REPOSITORY https://github.com/facebook/rocksdb.git GIT_TAG v8.1.1 SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" - BINARY_DIR "${TD_CONTRIB_DIR}/rocksdb" CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" From f2b438622a17cd76d5600289074fa192d52779c0 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 24 May 2023 19:47:56 +0800 Subject: [PATCH 089/238] skip some errors --- tests/pytest/crash_gen/crash_gen_main.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 9228cecdb5..5024f1e2fe 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2043,18 +2043,19 @@ class TdSuperTable: for topic in current_topic_list: topic_list.append(topic) - consumer.subscribe(topic_list) - - # consumer with random work life - time_start = time.time() - while 1: - res = consumer.poll(1) - consumer.commit(res) - if time.time() - time_start > random.randint(5, 50): - break try: + consumer.subscribe(topic_list) + + # consumer with random work life + time_start = time.time() + while 1: + res = consumer.poll(1) + consumer.commit(res) + if time.time() - time_start > random.randint(5, 50): + break consumer.unsubscribe() - except TmqError as e: + consumer.close() + except TmqError as err: # topic deleted by other threads pass return From 4a4e3d92758625d74dcc5529c158aa6720a7dc22 Mon Sep 17 00:00:00 2001 From: "chao.feng" Date: Tue, 23 May 2023 18:15:47 +0800 Subject: [PATCH 090/238] update replica number in case and only left the scenario of 3 replica --- tests/parallel_test/cases.task | 2 + tests/system-test/2-query/ts_3398.py | 56 ++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 tests/system-test/2-query/ts_3398.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index cf967c9553..350f14b05d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -342,6 +342,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3398.py -N 3 -n 3 + ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 diff --git a/tests/system-test/2-query/ts_3398.py b/tests/system-test/2-query/ts_3398.py new file mode 100644 index 0000000000..54d5c91804 --- /dev/null +++ b/tests/system-test/2-query/ts_3398.py @@ -0,0 +1,56 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.sqlset import * +import datetime + + +class TDTestCase: + """This test case is used to verify the aliasName of Node structure is not truncated + when sum clause is more than 65 bits. + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), False) + + def run(self): + # test case for https://jira.taosdata.com:18080/browse/TS-3405: + # create db + ret = tdSql.execute("CREATE DATABASE IF NOT EXISTS statistics1 REPLICA {} DURATION 14400m KEEP 5256000m,5256000m,5256000m PRECISION 'ms' MINROWS 100 MAXROWS 4096 COMP 2;".format(self.replicaVar)) + tdSql.execute("use statistics1;") + + # create stable + ret = tdSql.execute("CREATE STABLE IF NOT EXISTS statistics1.`g`(`day` timestamp,`run_state` tinyint) TAGS(`vin` binary(32));") + ret = tdSql.execute("CREATE STABLE IF NOT EXISTS statistics1.`b`(`day` timestamp, `total_heart` int) TAGS(`vin` binary(32));") + ret = tdSql.execute("CREATE STABLE IF NOT EXISTS statistics1.`tg`(`day` timestamp,`lt_4177` int,`f30_4177` int, `f35_4177` int) TAGS(`vin` binary(32));") + + # insert the data to table + ret = tdSql.execute("insert into d1001 using statistics1.`g` tags('NJHYNBSAS0000061') values (%s, %d)" % ("'2023-05-01'", 99)) + ret = tdSql.execute("insert into d2001 using statistics1.`b` tags('NJHYNBSAS0000061') values (%s, %d)" % ("'2023-05-01'", 99)) + ret = tdSql.execute("insert into d3001 using statistics1.`tg` tags('NJHYNBSAS0000061') values (%s, %d, %d, %d)" % ("'2023-05-01'", 99, 99, 99)) + + # execute the sql statements + ret = tdSql.query("SELECT b.`day` `day`,sum(CASE WHEN tg.lt_4177 IS NULL THEN 0 ELSE tg.lt_4177 END \ + + CASE WHEN tg.f35_4177 IS NULL THEN 0 ELSE tg.f35_4177 END) / 3600 es0,sum(CASE WHEN tg.lt_4177 \ + IS NULL THEN 0 ELSE tg.lt_4177 END + CASE WHEN tg.f35_4177 IS NULL THEN 0 ELSE tg.f35_4177 \ + END + CASE WHEN tg.f30_4177 IS NULL THEN 0 ELSE tg.f30_4177 END) / 3600 es1 FROM \ + statistics1.b b,statistics1.tg tg,statistics1.g g WHERE b.`day` = tg.`day` AND g.`day` = b.`day` \ + AND b.vin = tg.vin AND b.vin = g.vin AND b.`day` BETWEEN '2023-05-01' AND '2023-05-05' \ + AND b.vin = 'NJHYNBSAS0000061' AND g.vin IS NOT NULL AND b.vin IS NOT NULL AND tg.vin IS NOT NULL \ + GROUP BY b.`day`;") + # check the result + if 0.055 in tdSql.queryResult[0] and 0.0825 in tdSql.queryResult[0]: + tdLog.info("query result is correct") + else: + tdLog.info("query result is wrong") + + def stop(self): + # clear the db + tdSql.execute("drop database if exists statistics1;") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 459c1e6c0cbafc55bc05f34846442b14bc891bb1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 25 May 2023 13:30:43 +0800 Subject: [PATCH 091/238] fix: fix timestamp conversion error due to daylight saving time --- source/common/src/ttime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index dbb31374de..0aeb420b67 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -93,13 +93,13 @@ int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 'T'); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 'T'); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 'T'); } } else { if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 0); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 0); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 0); } } } From a57da3bdf791e0d9912d4b84dfeaa66595c88f0c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 25 May 2023 14:13:34 +0800 Subject: [PATCH 092/238] fix: fix log not compressed when logKeepDays set to positive value --- source/util/src/tlog.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e3b1c2fa2f..c405fb6a25 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -210,15 +210,15 @@ static void taosKeepOldLog(char *oldName) { (void)taosRenameFile(oldName, fileName); - if (tsLogKeepDays < 0) { - char compressFileName[LOG_FILE_NAME_LEN + 20]; - snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); - if (taosCompressFile(fileName, compressFileName) == 0) { - (void)taosRemoveFile(fileName); - } + char compressFileName[LOG_FILE_NAME_LEN + 20]; + snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); + if (taosCompressFile(fileName, compressFileName) == 0) { + (void)taosRemoveFile(fileName); } - taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays)); + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } } static void *taosThreadToOpenNewFile(void *param) { From 2f90fda13d4b2f029699d69d482ffe2565927ade Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 25 May 2023 16:15:10 +0800 Subject: [PATCH 093/238] change parameter --- source/libs/stream/src/streamBackendRocksdb.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 16ba81c74a..f9a9bef960 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -86,10 +86,10 @@ void* streamBackendInit(const char* path) { rocksdb_options_set_env(opts, env); rocksdb_options_set_create_if_missing(opts, 1); rocksdb_options_set_create_missing_column_families(opts, 1); - rocksdb_options_set_write_buffer_size(opts, 128 << 20); - rocksdb_options_set_max_total_wal_size(opts, 128 << 20); + rocksdb_options_set_write_buffer_size(opts, 64 << 20); + rocksdb_options_set_max_total_wal_size(opts, 64 << 20); rocksdb_options_set_recycle_log_file_num(opts, 6); - rocksdb_options_set_max_write_buffer_number(opts, 3); + rocksdb_options_set_max_write_buffer_number(opts, 2); rocksdb_options_set_info_log_level(opts, 0); pHandle->env = env; @@ -740,7 +740,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, SHashObj* ids) { qError("failed to open rocksdb cf, reason:%s", err); taosMemoryFree(err); } else { - qDebug("succ to open rocksdb cf, reason:%s", err); + qDebug("succ to open rocksdb cf, reason"); } pIter = taosHashIterate(ids, NULL); From 9cb4a114e2e0118c266eb068bfbd94f9c8cb59ab Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 25 May 2023 16:20:00 +0800 Subject: [PATCH 094/238] change parameter --- source/libs/stream/src/streamBackendRocksdb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f9a9bef960..f444dd01e3 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -80,14 +80,14 @@ void* streamBackendInit(const char* path) { rocksdb_env_set_low_priority_background_threads(env, 4); rocksdb_env_set_high_priority_background_threads(env, 2); - rocksdb_cache_t* cache = rocksdb_cache_create_lru(128 << 20); + rocksdb_cache_t* cache = rocksdb_cache_create_lru(64 << 20); rocksdb_options_t* opts = rocksdb_options_create(); rocksdb_options_set_env(opts, env); rocksdb_options_set_create_if_missing(opts, 1); rocksdb_options_set_create_missing_column_families(opts, 1); rocksdb_options_set_write_buffer_size(opts, 64 << 20); - rocksdb_options_set_max_total_wal_size(opts, 64 << 20); + rocksdb_options_set_max_total_wal_size(opts, 128 << 20); rocksdb_options_set_recycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 2); rocksdb_options_set_info_log_level(opts, 0); From 5933f85b56461685cecdaade4e0b1d804f96829e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 25 May 2023 17:51:33 +0800 Subject: [PATCH 095/238] change parameter --- cmake/cmake.define | 6 +++--- contrib/CMakeLists.txt | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 0d5c21604a..a233834fbe 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0) -set(CMAKE_VERBOSE_MAKEFILE OFF) +set(CMAKE_VERBOSE_MAKEFILE ON) set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory @@ -123,8 +123,8 @@ ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ELSE () - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -g3 -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -O3 -s -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ENDIF () # disable all assert diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b3333dee99..842b74098a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -231,6 +231,7 @@ if(${BUILD_WITH_ROCKSDB}) if(${TD_LINUX}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result") endif(${TD_LINUX}) + MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) if(${TD_DARWIN}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") @@ -248,7 +249,7 @@ if(${BUILD_WITH_ROCKSDB}) endif(${TD_DARWIN}) if(${TD_WINDOWS}) - option(WITH_JNI "" ON) + option(WITH_JNI "" OFF) endif(${TD_WINDOWS}) if(${TD_WINDOWS}) @@ -260,7 +261,7 @@ if(${BUILD_WITH_ROCKSDB}) option(WITH_FALLOCATE "" OFF) option(WITH_JEMALLOC "" OFF) option(WITH_GFLAGS "" OFF) - option(PORTABLE "" ON) + option(PORTABLE "" OFF) option(WITH_LIBURING "" OFF) option(FAIL_ON_WARNINGS OFF) From cd44bd9f3f3740c8f37f968ee9955a2383c72f51 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 25 May 2023 18:16:33 +0800 Subject: [PATCH 096/238] change parameter --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f444dd01e3..11e33da5a4 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -86,7 +86,7 @@ void* streamBackendInit(const char* path) { rocksdb_options_set_env(opts, env); rocksdb_options_set_create_if_missing(opts, 1); rocksdb_options_set_create_missing_column_families(opts, 1); - rocksdb_options_set_write_buffer_size(opts, 64 << 20); + rocksdb_options_set_write_buffer_size(opts, 48 << 20); rocksdb_options_set_max_total_wal_size(opts, 128 << 20); rocksdb_options_set_recycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 2); From abca0495b113f901f19d848e110f646c3c0c7607 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Thu, 25 May 2023 18:54:22 +0800 Subject: [PATCH 097/238] enh(taosAdapter): change taosAdapter to main branch --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index c67918351d..13826a1a74 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 565ca21 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 2713f4f6d44eb0ed996995a43943dc66b913e024 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 25 May 2023 19:01:58 +0800 Subject: [PATCH 098/238] feat: support create topic as stable with conditions --- include/common/ttokendef.h | 1 + include/libs/nodes/cmdnodes.h | 1 + include/libs/qcom/query.h | 7 + source/common/src/tmsg.c | 8 +- source/libs/parser/inc/parAst.h | 2 +- source/libs/parser/inc/parUtil.h | 1 + source/libs/parser/src/parAstCreater.c | 40 +- source/libs/parser/src/parAstParser.c | 5 + source/libs/parser/src/parTranslater.c | 106 +- source/libs/parser/src/parUtil.c | 16 + source/libs/parser/src/sql.c | 1764 +++++++++---------- source/libs/parser/test/parInitialCTest.cpp | 9 + source/libs/qcom/src/queryUtil.c | 12 + tests/script/tsim/tmq/topic.sim | 10 + 14 files changed, 1059 insertions(+), 923 deletions(-) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 10b5328e6d..d1a8f6d0c6 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -347,6 +347,7 @@ #define TK_WAL 329 + #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 #define TK_NK_ILLEGAL 602 diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 2323d044ec..0c1946163a 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -358,6 +358,7 @@ typedef struct SCreateTopicStmt { bool ignoreExists; bool withMeta; SNode* pQuery; + SNode* pWhere; } SCreateTopicStmt; typedef struct SDropTopicStmt { diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 3841210076..5abb491d3e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -51,6 +51,12 @@ typedef enum { TARGET_TYPE_OTHER, } ETargetType; +typedef enum { + TCOL_TYPE_COLUMN = 1, + TCOL_TYPE_TAG, + TCOL_TYPE_NONE, +} ETableColumnType; + #define QUERY_POLICY_VNODE 1 #define QUERY_POLICY_HYBRID 2 #define QUERY_POLICY_QNODE 3 @@ -257,6 +263,7 @@ void destroyQueryExecRes(SExecResult* pRes); int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len); char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); +void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst); void freeVgInfo(SDBVgInfo* vgInfo); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index cb4d0c9034..bc826b38da 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3836,8 +3836,12 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { if (tEncodeCStr(&encoder, pReq->subStbName) < 0) return -1; } - if (tEncodeI32(&encoder, strlen(pReq->ast)) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->ast) < 0) return -1; + if (pReq->ast && strlen(pReq->ast) > 0) { + if (tEncodeI32(&encoder, strlen(pReq->ast)) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->ast) < 0) return -1; + } else { + if (tEncodeI32(&encoder, 0) < 0) return -1; + } } if (tEncodeI32(&encoder, strlen(pReq->sql)) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index e08b77e681..9cfb469e68 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -206,7 +206,7 @@ SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName, bool withMeta); SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, - bool withMeta); + bool withMeta, SNode* pWhere); SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName); SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, SToken* pTopicName); SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 850571eea1..9632ccf0fb 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -115,6 +115,7 @@ int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes); void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request); +SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 7258f6c415..5ebafaa9f9 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -822,16 +822,9 @@ SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill) { SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { CHECK_PARSER_STATUS(pCxt); - SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + SNode* select = createSelectStmtImpl(isDistinct, pProjectionList, pTable); CHECK_OUT_OF_MEM(select); - select->isDistinct = isDistinct; - select->pProjectionList = pProjectionList; - select->pFromTable = pTable; - sprintf(select->stmtName, "%p", select); - select->isTimeLineResult = true; - select->onlyHasKeepOrderFunc = true; - select->timeRange = TSWINDOW_INITIALIZER; - return (SNode*)select; + return select; } static void setSubquery(SNode* pStmt) { @@ -1703,7 +1696,7 @@ SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, ST } SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable, - bool withMeta) { + bool withMeta, SNode* pWhere) { CHECK_PARSER_STATUS(pCxt); if (!checkTopicName(pCxt, pTopicName)) { return NULL; @@ -1713,32 +1706,7 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); pStmt->ignoreExists = ignoreExists; pStmt->withMeta = withMeta; - -// SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); -// if (NULL == pSelect) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// sprintf(pSelect->stmtName, "%p", pSelect); -// -// SRealTableNode* pRealTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); -// if (NULL == pRealTable) { -// nodesDestroyNode((SNode*)pSelect); -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// snprintf(pRealTable->table.dbName, sizeof(pRealTable->table.dbName), "%s", pDb); -// snprintf(pRealTable->table.tableName, sizeof(pRealTable->table.tableName), "%s", pTable); -// snprintf(pRealTable->table.tableAlias, sizeof(pRealTable->table.tableAlias), "%s", pTable); -// pSelect->pFromTable = (SNode*)pRealTable; -// -// if (numOfProjs >= 0) { -// pSelect->pProjectionList = createProjectCols(numOfProjs, pProjCol); -// if (NULL == pSelect->pProjectionList) { -// nodesDestroyNode((SNode*)pSelect); -// return TSDB_CODE_OUT_OF_MEMORY; -// } -// } -// -// pStmt->pQuery = pSelect; + pStmt->pWhere = pWhere; strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName); strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5db1f5dbdb..dea0c4bd50 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -355,6 +355,11 @@ static int32_t collectMetaKeyFromCreateTopic(SCollectMetaKeyCxt* pCxt, SCreateTo if (NULL != pStmt->pQuery) { return collectMetaKeyFromQuery(pCxt, pStmt->pQuery); } + if (NULL != pStmt->pWhere) { + int32_t code = collectMetaKeyFromRealTableImpl(pCxt, pStmt->subDbName, pStmt->subSTbName, + AUTH_TYPE_READ); + return code; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7c11443acb..6de1bd4b85 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -55,6 +55,13 @@ typedef struct STranslateContext { bool showRewrite; } STranslateContext; +typedef struct SBuildTopicContext { + bool colExists; + bool colNotFound; + STableMeta* pMeta; + SNodeList* pTags; +} SBuildTopicContext; + typedef struct SFullDatabaseName { char fullDbName[TSDB_DB_FNAME_LEN]; } SFullDatabaseName; @@ -5789,12 +5796,107 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS return code; } +static int32_t addTagList(SNodeList** ppList, SNode* pNode) { + if (NULL == *ppList) { + *ppList = nodesMakeList(); + } + + nodesListStrictAppend(*ppList, pNode); + + return TSDB_CODE_SUCCESS; +} + +static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) { + SBuildTopicContext* pCxt = (SBuildTopicContext*)pContext; + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + ETableColumnType type; + getColumnTypeFromMeta(pCxt->pMeta, ((SColumnNode*)pNode)->colName, &type); + if (type == TCOL_TYPE_COLUMN) { + pCxt->colExists = true; + return DEAL_RES_ERROR; + } else if (type == TCOL_TYPE_TAG) { + addTagList(&pCxt->pTags, nodesCloneNode(pNode)); + } else { + pCxt->colNotFound = true; + return DEAL_RES_ERROR; + } + } else if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + if (0 == strcasecmp(pFunc->functionName, "tbname")) { + addTagList(&pCxt->pTags, nodesCloneNode(pNode)); + } + } + + return DEAL_RES_CONTINUE; +} + +static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta, SNodeList** ppProjection) { + SBuildTopicContext colCxt = {.colExists = false, .colNotFound = false, .pMeta = pMeta, .pTags = NULL}; + nodesWalkExprPostOrder(pStmt->pWhere, checkColumnTagsInCond, &colCxt); + if (colCxt.colNotFound) { + nodesDestroyList(colCxt.pTags); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Invalid column name"); + } else if (colCxt.colExists) { + nodesDestroyList(colCxt.pTags); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Columns are forbidden in where clause"); + } + if (NULL == colCxt.pTags) { + for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) { + SSchema* tag = &pMeta->schema[pMeta->tableInfo.numOfColumns + i]; + SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + strcpy(col->colName, tag->name); + strcpy(col->node.aliasName, col->colName); + strcpy(col->node.userAlias, col->colName); + addTagList(&colCxt.pTags, (SNode*)col); + } + } + + *ppProjection = colCxt.pTags; + return TSDB_CODE_SUCCESS; +} + +static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SNode** pSelect) { + SParseContext* pParCxt = pCxt->pParseCxt; + SRequestConnInfo connInfo = {.pTrans = pParCxt->pTransporter, + .requestId = pParCxt->requestId, + .requestObjRefId = pParCxt->requestRid, + .mgmtEps = pParCxt->mgmtEpSet}; + SName name; + STableMeta* pMeta = NULL; + int32_t code = getTableMetaImpl(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta); + if (code) { + taosMemoryFree(pMeta); + return code; + } + if (TSDB_SUPER_TABLE != pMeta->tableType) { + taosMemoryFree(pMeta); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Only supertable table can be used"); + } + + SNodeList* pProjection = NULL; + code = checkCollectTopicTags(pCxt, pStmt, pMeta, &pProjection); + if (TSDB_CODE_SUCCESS == code) { + SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + strcpy(realTable->table.dbName, pStmt->subDbName); + strcpy(realTable->table.tableName, pStmt->subSTbName); + strcpy(realTable->table.tableAlias, pStmt->subSTbName); + *pSelect = createSelectStmtImpl(true, pProjection, (SNode*)realTable); + ((SSelectStmt*)*pSelect)->pWhere = nodesCloneNode(pStmt->pWhere); + code = translateQuery(pCxt, *pSelect); + } + + taosMemoryFree(pMeta); + return code; +} + static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) { - if (NULL == pStmt->pQuery) { + if (NULL == pStmt->pQuery && NULL == pStmt->pWhere) { return TSDB_CODE_SUCCESS; } - if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { + if (pStmt->pWhere) { + return buildQueryForTableTopic(pCxt, pStmt, &pStmt->pQuery); + } else if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (!pSelect->isDistinct && (NULL != pSelect->pFromTable && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable)) && diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 14da6f8aab..644d1cca05 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -666,6 +666,22 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog return code; } + +SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { + SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == select) { + return NULL; + } + select->isDistinct = isDistinct; + select->pProjectionList = pProjectionList; + select->pFromTable = pTable; + sprintf(select->stmtName, "%p", select); + select->isTimeLineResult = true; + select->onlyHasKeepOrderFunc = true; + select->timeRange = TSWINDOW_INITIALIZER; + return (SNode*)select; +} + static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj** pHash) { if (NULL == *pHash) { *pHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 2e473455e4..41a4327842 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -140,18 +140,18 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 762 +#define YYNSTATE 764 #define YYNRULE 583 #define YYNRULE_WITH_ACTION 583 #define YYNTOKEN 330 -#define YY_MAX_SHIFT 761 -#define YY_MIN_SHIFTREDUCE 1136 -#define YY_MAX_SHIFTREDUCE 1718 -#define YY_ERROR_ACTION 1719 -#define YY_ACCEPT_ACTION 1720 -#define YY_NO_ACTION 1721 -#define YY_MIN_REDUCE 1722 -#define YY_MAX_REDUCE 2304 +#define YY_MAX_SHIFT 763 +#define YY_MIN_SHIFTREDUCE 1138 +#define YY_MAX_SHIFTREDUCE 1720 +#define YY_ERROR_ACTION 1721 +#define YY_ACCEPT_ACTION 1722 +#define YY_NO_ACTION 1723 +#define YY_MIN_REDUCE 1724 +#define YY_MAX_REDUCE 2306 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -218,609 +218,609 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2904) +#define YY_ACTTAB_COUNT (2902) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2116, 2010, 1881, 396, 434, 168, 636, 1734, 433, 1989, - /* 10 */ 672, 162, 46, 44, 1646, 1723, 2008, 642, 374, 1894, - /* 20 */ 393, 506, 1495, 440, 39, 38, 1942, 503, 45, 43, - /* 30 */ 42, 41, 40, 1576, 1791, 1493, 123, 2134, 1520, 122, - /* 40 */ 121, 120, 119, 118, 117, 116, 115, 114, 2279, 2084, - /* 50 */ 593, 671, 593, 2275, 194, 2275, 39, 38, 167, 1571, - /* 60 */ 45, 43, 42, 41, 40, 19, 1833, 340, 2281, 186, - /* 70 */ 2281, 186, 1501, 2276, 619, 2276, 619, 45, 43, 42, - /* 80 */ 41, 40, 2115, 501, 654, 2151, 502, 1758, 169, 2117, - /* 90 */ 675, 2119, 2120, 670, 182, 665, 1520, 758, 36, 298, - /* 100 */ 15, 735, 734, 733, 732, 405, 1931, 731, 730, 144, - /* 110 */ 725, 724, 723, 722, 721, 720, 719, 157, 715, 714, - /* 120 */ 713, 404, 403, 710, 709, 708, 175, 174, 594, 2241, - /* 130 */ 509, 654, 1320, 502, 1758, 123, 1578, 1579, 122, 121, - /* 140 */ 120, 119, 118, 117, 116, 115, 114, 1311, 697, 696, - /* 150 */ 695, 1315, 694, 1317, 1318, 693, 690, 179, 1326, 687, - /* 160 */ 1328, 1329, 684, 681, 49, 640, 1551, 1561, 654, 2219, - /* 170 */ 655, 1892, 1577, 1580, 1720, 39, 38, 360, 1993, 45, - /* 180 */ 43, 42, 41, 40, 1409, 1410, 1496, 1523, 1494, 133, - /* 190 */ 655, 1892, 387, 62, 1715, 2216, 539, 39, 38, 280, - /* 200 */ 165, 45, 43, 42, 41, 40, 39, 38, 1894, 191, - /* 210 */ 45, 43, 42, 41, 40, 1499, 1500, 1868, 1550, 1553, - /* 220 */ 1554, 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, 1569, - /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 46, 44, 1722, 179, - /* 240 */ 62, 343, 93, 1518, 393, 408, 1495, 62, 49, 407, - /* 250 */ 471, 2116, 2280, 485, 352, 2275, 484, 1576, 1989, 1493, - /* 260 */ 1994, 672, 132, 131, 130, 129, 128, 127, 126, 125, - /* 270 */ 124, 2279, 454, 1522, 486, 2276, 2278, 456, 432, 396, - /* 280 */ 431, 707, 2280, 1571, 1275, 2275, 211, 165, 2134, 19, - /* 290 */ 504, 593, 1765, 1714, 2275, 1894, 1501, 1274, 1605, 1685, - /* 300 */ 2084, 2279, 671, 196, 2010, 2276, 2277, 430, 618, 2281, - /* 310 */ 186, 2275, 518, 2116, 2276, 619, 386, 621, 66, 2007, - /* 320 */ 642, 758, 361, 672, 15, 1767, 617, 186, 1264, 655, - /* 330 */ 1892, 2276, 619, 2115, 444, 257, 2151, 655, 1892, 110, - /* 340 */ 2117, 675, 2119, 2120, 670, 189, 665, 1521, 133, 143, - /* 350 */ 2134, 150, 2175, 2204, 1606, 544, 55, 389, 2200, 487, - /* 360 */ 1578, 1579, 2084, 482, 671, 1266, 476, 475, 474, 473, - /* 370 */ 470, 469, 468, 467, 466, 462, 461, 460, 459, 342, - /* 380 */ 451, 450, 449, 613, 446, 445, 359, 1650, 1520, 1521, - /* 390 */ 1551, 1561, 189, 1520, 545, 2115, 1577, 1580, 2151, 189, - /* 400 */ 189, 110, 2117, 675, 2119, 2120, 670, 707, 665, 1191, - /* 410 */ 1496, 1190, 1494, 2295, 62, 2204, 1262, 630, 140, 389, - /* 420 */ 2200, 279, 50, 608, 641, 630, 140, 1745, 35, 391, - /* 430 */ 1600, 1601, 1602, 1603, 1604, 1608, 1609, 1610, 1611, 1499, - /* 440 */ 1500, 1192, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, - /* 450 */ 1560, 667, 663, 1569, 1570, 1572, 1573, 1574, 1575, 2, - /* 460 */ 12, 46, 44, 1354, 1355, 1495, 402, 401, 62, 393, - /* 470 */ 1279, 1495, 557, 556, 555, 516, 2084, 2003, 1493, 547, - /* 480 */ 137, 551, 1576, 1278, 1493, 550, 2280, 2116, 32, 1502, - /* 490 */ 549, 554, 368, 367, 39, 38, 548, 672, 45, 43, - /* 500 */ 42, 41, 40, 1522, 1523, 1426, 1427, 2116, 1571, 614, - /* 510 */ 609, 602, 1674, 699, 19, 1501, 1935, 669, 185, 2212, - /* 520 */ 2213, 1501, 138, 2217, 2134, 632, 184, 2212, 2213, 398, - /* 530 */ 138, 2217, 1937, 1939, 2051, 1191, 2084, 1190, 671, 1552, - /* 540 */ 758, 1425, 1428, 1519, 2134, 489, 758, 39, 38, 15, - /* 550 */ 228, 45, 43, 42, 41, 40, 2084, 153, 671, 605, - /* 560 */ 604, 1672, 1673, 1675, 1676, 1677, 189, 1192, 166, 2115, - /* 570 */ 1744, 580, 2151, 318, 189, 170, 2117, 675, 2119, 2120, - /* 580 */ 670, 246, 665, 28, 1708, 1578, 1579, 316, 73, 2115, - /* 590 */ 12, 72, 2151, 2094, 562, 334, 2117, 675, 2119, 2120, - /* 600 */ 670, 668, 665, 656, 2169, 655, 1892, 2102, 107, 572, - /* 610 */ 209, 497, 495, 492, 54, 1551, 1561, 2098, 612, 2084, - /* 620 */ 189, 1577, 1580, 242, 438, 141, 620, 2296, 213, 1496, - /* 630 */ 478, 1494, 504, 1884, 1765, 1496, 1794, 1494, 165, 565, - /* 640 */ 1944, 1944, 1505, 641, 559, 2134, 1895, 358, 373, 241, - /* 650 */ 62, 279, 195, 2100, 376, 1942, 1942, 1501, 1499, 1500, - /* 660 */ 1169, 289, 290, 665, 1499, 1500, 288, 1550, 1553, 1554, - /* 670 */ 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, 1569, 1570, - /* 680 */ 1572, 1573, 1574, 1575, 2, 46, 44, 1581, 109, 70, - /* 690 */ 202, 201, 69, 393, 639, 1495, 2003, 2116, 611, 1171, - /* 700 */ 1974, 1174, 1175, 557, 556, 555, 1576, 633, 1493, 365, - /* 710 */ 547, 137, 551, 477, 641, 698, 550, 618, 655, 1892, - /* 720 */ 2275, 549, 554, 368, 367, 630, 140, 548, 81, 80, - /* 730 */ 437, 1643, 1571, 193, 2134, 617, 186, 439, 1938, 1939, - /* 740 */ 2276, 619, 717, 244, 1944, 1501, 2084, 243, 671, 655, - /* 750 */ 1892, 383, 14, 13, 341, 655, 1892, 423, 1877, 1942, - /* 760 */ 421, 417, 413, 410, 430, 650, 1743, 2003, 448, 1944, - /* 770 */ 758, 106, 2077, 47, 463, 366, 388, 364, 363, 2115, - /* 780 */ 541, 103, 2151, 2116, 1942, 110, 2117, 675, 2119, 2120, - /* 790 */ 670, 1944, 665, 672, 1869, 655, 1892, 183, 397, 2204, - /* 800 */ 101, 543, 189, 389, 2200, 542, 1942, 90, 347, 1578, - /* 810 */ 1579, 372, 87, 573, 464, 2084, 188, 2078, 593, 1735, - /* 820 */ 2134, 2275, 553, 552, 2230, 1885, 277, 2212, 629, 362, - /* 830 */ 134, 628, 2084, 2275, 671, 1742, 2281, 186, 1887, 1551, - /* 840 */ 1561, 2276, 619, 630, 140, 1577, 1580, 245, 617, 186, - /* 850 */ 655, 1892, 1619, 2276, 619, 458, 1586, 2219, 12, 1496, - /* 860 */ 10, 1494, 1520, 593, 457, 2115, 2275, 1741, 2151, 517, - /* 870 */ 1879, 111, 2117, 675, 2119, 2120, 670, 657, 665, 2176, - /* 880 */ 622, 2281, 186, 2215, 2084, 2204, 2276, 619, 1499, 1500, - /* 890 */ 2201, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, - /* 900 */ 667, 663, 1569, 1570, 1572, 1573, 1574, 1575, 2, 46, - /* 910 */ 44, 659, 1523, 2176, 1468, 1469, 2084, 393, 427, 1495, - /* 920 */ 52, 2116, 3, 705, 155, 154, 702, 701, 700, 152, - /* 930 */ 1576, 633, 1493, 39, 38, 655, 1892, 45, 43, 42, - /* 940 */ 41, 40, 429, 425, 187, 2212, 2213, 2116, 138, 2217, - /* 950 */ 198, 1875, 655, 1892, 1889, 578, 1571, 672, 2134, 2238, - /* 960 */ 705, 155, 154, 702, 701, 700, 152, 399, 1520, 1501, - /* 970 */ 2084, 247, 671, 655, 1892, 165, 42, 41, 40, 2219, - /* 980 */ 1642, 39, 38, 1894, 2134, 45, 43, 42, 41, 40, - /* 990 */ 84, 543, 589, 83, 758, 542, 2084, 47, 671, 655, - /* 1000 */ 1892, 593, 1740, 2115, 2275, 2214, 2151, 2116, 1552, 110, - /* 1010 */ 2117, 675, 2119, 2120, 670, 2067, 665, 672, 634, 2281, - /* 1020 */ 186, 183, 9, 2204, 2276, 619, 1739, 389, 2200, 2115, - /* 1030 */ 1867, 87, 2151, 1578, 1579, 110, 2117, 675, 2119, 2120, - /* 1040 */ 670, 2094, 665, 254, 2134, 1738, 571, 2295, 2231, 2204, - /* 1050 */ 146, 2084, 135, 389, 2200, 1883, 2084, 1888, 671, 569, - /* 1060 */ 256, 567, 1989, 1551, 1561, 2098, 34, 655, 1892, 1577, - /* 1070 */ 1580, 1607, 39, 38, 1662, 2084, 45, 43, 42, 41, - /* 1080 */ 40, 729, 727, 1496, 1896, 1494, 638, 655, 1892, 2115, - /* 1090 */ 655, 1892, 2151, 164, 2084, 169, 2117, 675, 2119, 2120, - /* 1100 */ 670, 2100, 665, 1174, 1175, 312, 293, 200, 1921, 652, - /* 1110 */ 623, 665, 1499, 1500, 1552, 1550, 1553, 1554, 1555, 1556, - /* 1120 */ 1557, 1558, 1559, 1560, 667, 663, 1569, 1570, 1572, 1573, - /* 1130 */ 1574, 1575, 2, 46, 44, 1737, 2242, 2116, 655, 1892, - /* 1140 */ 1736, 393, 1733, 1495, 2094, 33, 1732, 672, 1731, 2251, - /* 1150 */ 655, 1892, 655, 1892, 1576, 1612, 1493, 653, 2102, 705, - /* 1160 */ 155, 154, 702, 701, 700, 152, 1730, 142, 2098, 299, - /* 1170 */ 2175, 400, 2224, 1639, 2134, 1944, 703, 1177, 704, 1935, - /* 1180 */ 1571, 1935, 1729, 1519, 2084, 2094, 2084, 1728, 671, 2084, - /* 1190 */ 1943, 2084, 2094, 1501, 1870, 2084, 625, 2084, 1727, 2102, - /* 1200 */ 1726, 1725, 2070, 251, 2100, 377, 2103, 718, 234, 2098, - /* 1210 */ 1854, 232, 74, 592, 665, 2084, 2098, 236, 758, 2115, - /* 1220 */ 235, 15, 2151, 2116, 422, 110, 2117, 675, 2119, 2120, - /* 1230 */ 670, 2084, 665, 672, 1639, 600, 2084, 2295, 148, 2204, - /* 1240 */ 153, 441, 546, 389, 2200, 2100, 390, 2084, 1781, 2084, - /* 1250 */ 2084, 415, 2100, 238, 442, 665, 237, 1578, 1579, 1774, - /* 1260 */ 2134, 82, 665, 240, 1260, 153, 239, 1772, 711, 575, - /* 1270 */ 558, 574, 2084, 662, 671, 64, 255, 64, 261, 1717, - /* 1280 */ 1718, 560, 14, 13, 666, 2105, 1834, 1551, 1561, 563, - /* 1290 */ 1240, 2244, 274, 1577, 1580, 227, 268, 1463, 606, 1832, - /* 1300 */ 1504, 1503, 712, 1831, 153, 2115, 48, 1496, 2151, 1494, - /* 1310 */ 2135, 110, 2117, 675, 2119, 2120, 670, 286, 665, 91, - /* 1320 */ 71, 151, 1466, 2295, 1238, 2204, 406, 64, 48, 389, - /* 1330 */ 2200, 1998, 1671, 53, 1670, 263, 1499, 1500, 2107, 1550, - /* 1340 */ 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 667, 663, - /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 2116, 402, 401, - /* 1360 */ 1221, 637, 153, 1423, 48, 679, 1768, 672, 1509, 2269, - /* 1370 */ 151, 1764, 153, 1759, 291, 1932, 2234, 647, 295, 1576, - /* 1380 */ 136, 1502, 631, 2116, 1613, 1562, 276, 151, 273, 1, - /* 1390 */ 5, 409, 356, 672, 2134, 2223, 414, 1222, 1446, 306, - /* 1400 */ 199, 443, 1523, 1999, 480, 1571, 2084, 447, 671, 452, - /* 1410 */ 1518, 490, 465, 1991, 479, 472, 753, 491, 1501, 1305, - /* 1420 */ 2134, 311, 1332, 1597, 481, 488, 203, 1336, 626, 1343, - /* 1430 */ 2116, 204, 2084, 493, 671, 494, 206, 1341, 496, 2115, - /* 1440 */ 672, 498, 2151, 661, 156, 110, 2117, 675, 2119, 2120, - /* 1450 */ 670, 2116, 665, 1507, 1506, 1524, 4, 2295, 499, 2204, - /* 1460 */ 507, 672, 1526, 389, 2200, 2115, 500, 2134, 2151, 508, - /* 1470 */ 510, 110, 2117, 675, 2119, 2120, 670, 1521, 665, 2084, - /* 1480 */ 1525, 671, 214, 2295, 511, 2204, 512, 216, 2134, 389, - /* 1490 */ 2200, 1527, 513, 1194, 219, 515, 221, 85, 86, 519, - /* 1500 */ 2084, 536, 671, 225, 538, 537, 112, 346, 540, 1882, - /* 1510 */ 231, 1878, 2115, 2060, 577, 2151, 2057, 2056, 170, 2117, - /* 1520 */ 675, 2119, 2120, 670, 579, 665, 233, 89, 149, 307, - /* 1530 */ 158, 159, 1510, 2115, 1505, 248, 2151, 1880, 1876, 110, - /* 1540 */ 2117, 675, 2119, 2120, 670, 160, 665, 161, 583, 252, - /* 1550 */ 582, 2179, 1453, 2204, 584, 2116, 597, 389, 2200, 607, - /* 1560 */ 587, 1513, 1515, 250, 2250, 672, 590, 645, 259, 588, - /* 1570 */ 2297, 2235, 2245, 603, 663, 1569, 1570, 1572, 1573, 1574, - /* 1580 */ 1575, 262, 2249, 8, 616, 378, 610, 2226, 598, 596, - /* 1590 */ 595, 267, 2134, 379, 139, 272, 1639, 2298, 627, 1522, - /* 1600 */ 624, 2220, 382, 635, 2084, 269, 671, 1528, 2004, 308, - /* 1610 */ 643, 281, 96, 309, 644, 648, 98, 2018, 2017, 2016, - /* 1620 */ 649, 2116, 173, 385, 270, 1893, 271, 100, 61, 102, - /* 1630 */ 2185, 672, 1936, 313, 1855, 310, 754, 2115, 302, 755, - /* 1640 */ 2151, 757, 348, 110, 2117, 675, 2119, 2120, 670, 2116, - /* 1650 */ 665, 2274, 677, 275, 337, 2177, 51, 2204, 2134, 672, - /* 1660 */ 322, 389, 2200, 336, 326, 317, 2076, 2075, 349, 2074, - /* 1670 */ 2084, 315, 671, 78, 2071, 411, 412, 1486, 1487, 192, - /* 1680 */ 416, 2116, 2069, 418, 419, 420, 2134, 2068, 357, 2066, - /* 1690 */ 424, 672, 426, 2064, 428, 79, 1449, 1448, 2084, 2065, - /* 1700 */ 671, 2030, 2029, 2115, 2028, 435, 2151, 436, 2027, 110, - /* 1710 */ 2117, 675, 2119, 2120, 670, 2026, 665, 1982, 2134, 1400, - /* 1720 */ 1981, 658, 1979, 2204, 1978, 145, 1977, 389, 2200, 1980, - /* 1730 */ 2084, 2115, 671, 1976, 2151, 1975, 1973, 111, 2117, 675, - /* 1740 */ 2119, 2120, 670, 1972, 665, 1971, 197, 453, 1970, 455, - /* 1750 */ 1984, 2204, 2116, 1969, 1968, 2203, 2200, 483, 147, 1954, - /* 1760 */ 1953, 1952, 672, 2115, 1967, 1966, 2151, 1965, 1964, 111, - /* 1770 */ 2117, 675, 2119, 2120, 670, 1963, 665, 1962, 1961, 1960, - /* 1780 */ 2116, 1959, 1958, 2204, 1957, 1956, 1955, 660, 2200, 2134, - /* 1790 */ 672, 1983, 1951, 1950, 1402, 1949, 1948, 1947, 1946, 2116, - /* 1800 */ 1945, 2084, 1276, 671, 344, 1280, 345, 1797, 205, 669, - /* 1810 */ 1796, 1272, 1795, 1793, 207, 2116, 1754, 2134, 180, 1176, - /* 1820 */ 2104, 208, 1753, 2047, 2037, 672, 76, 2025, 77, 2084, - /* 1830 */ 210, 671, 2024, 218, 673, 220, 2134, 2151, 2002, 1871, - /* 1840 */ 111, 2117, 675, 2119, 2120, 670, 181, 665, 2084, 505, - /* 1850 */ 671, 212, 2134, 1792, 2204, 1790, 1214, 384, 351, 2200, - /* 1860 */ 520, 1788, 2115, 521, 2084, 2151, 671, 522, 328, 2117, - /* 1870 */ 675, 2119, 2120, 670, 526, 665, 2116, 524, 525, 1786, - /* 1880 */ 528, 2115, 529, 530, 2151, 1784, 672, 334, 2117, 675, - /* 1890 */ 2119, 2120, 670, 532, 665, 533, 2170, 2115, 581, 2116, - /* 1900 */ 2151, 534, 1771, 335, 2117, 675, 2119, 2120, 670, 672, - /* 1910 */ 665, 615, 1770, 2134, 1750, 1873, 761, 63, 392, 1348, - /* 1920 */ 1347, 1872, 230, 1263, 1261, 2084, 1259, 671, 1258, 1257, - /* 1930 */ 305, 1256, 1250, 1782, 1255, 369, 2134, 561, 1252, 726, - /* 1940 */ 1251, 394, 1249, 1775, 728, 370, 178, 1773, 2084, 371, - /* 1950 */ 671, 564, 751, 747, 743, 739, 303, 566, 2115, 1749, - /* 1960 */ 1748, 2151, 1747, 568, 335, 2117, 675, 2119, 2120, 670, - /* 1970 */ 570, 665, 113, 1473, 1475, 1472, 1477, 27, 2046, 1455, - /* 1980 */ 67, 2115, 2036, 1457, 2151, 56, 585, 335, 2117, 675, - /* 1990 */ 2119, 2120, 670, 2116, 665, 163, 108, 2023, 586, 296, - /* 2000 */ 2021, 253, 1459, 672, 375, 2280, 29, 20, 17, 591, - /* 2010 */ 1687, 258, 59, 6, 7, 599, 601, 60, 265, 260, - /* 2020 */ 1669, 31, 266, 2105, 171, 2116, 264, 21, 65, 30, - /* 2030 */ 2134, 651, 1661, 1707, 92, 672, 1708, 22, 1702, 278, - /* 2040 */ 1701, 380, 2084, 2116, 671, 1706, 1705, 381, 176, 2022, - /* 2050 */ 1636, 1635, 58, 672, 2020, 2019, 2001, 18, 94, 95, - /* 2060 */ 284, 646, 2134, 2000, 103, 97, 283, 23, 57, 297, - /* 2070 */ 285, 282, 1667, 294, 2084, 576, 671, 24, 2151, 287, - /* 2080 */ 2134, 330, 2117, 675, 2119, 2120, 670, 292, 665, 68, - /* 2090 */ 2116, 249, 2084, 11, 671, 99, 1588, 1587, 13, 1511, - /* 2100 */ 672, 1543, 1598, 678, 177, 190, 395, 2115, 1566, 2154, - /* 2110 */ 2151, 682, 1325, 319, 2117, 675, 2119, 2120, 670, 664, - /* 2120 */ 665, 685, 1564, 37, 16, 2115, 676, 2134, 2151, 1563, - /* 2130 */ 25, 320, 2117, 675, 2119, 2120, 670, 2116, 665, 2084, - /* 2140 */ 1535, 671, 674, 26, 688, 1333, 680, 672, 1330, 1327, - /* 2150 */ 683, 686, 691, 1321, 1319, 300, 1324, 1323, 1322, 2116, - /* 2160 */ 689, 692, 104, 1342, 105, 75, 1338, 1212, 706, 672, - /* 2170 */ 1244, 1243, 2115, 301, 2134, 2151, 1242, 1241, 321, 2117, - /* 2180 */ 675, 2119, 2120, 670, 2116, 665, 2084, 1239, 671, 1237, - /* 2190 */ 1236, 1235, 1270, 716, 672, 1233, 2134, 1232, 1231, 1230, - /* 2200 */ 1229, 1228, 1227, 1267, 1265, 1224, 2116, 1223, 2084, 1220, - /* 2210 */ 671, 1219, 1218, 1217, 1789, 736, 672, 737, 738, 2115, - /* 2220 */ 1787, 2134, 2151, 740, 741, 327, 2117, 675, 2119, 2120, - /* 2230 */ 670, 742, 665, 2084, 1785, 671, 744, 746, 745, 1783, - /* 2240 */ 748, 2115, 749, 2134, 2151, 750, 1769, 331, 2117, 675, - /* 2250 */ 2119, 2120, 670, 752, 665, 2084, 1166, 671, 304, 756, - /* 2260 */ 1721, 1721, 1497, 314, 759, 760, 2115, 1746, 1721, 2151, - /* 2270 */ 2116, 1721, 323, 2117, 675, 2119, 2120, 670, 1721, 665, - /* 2280 */ 672, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 1721, - /* 2290 */ 2116, 2151, 1721, 1721, 332, 2117, 675, 2119, 2120, 670, - /* 2300 */ 672, 665, 1721, 1721, 1721, 2116, 1721, 2134, 1721, 1721, - /* 2310 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 2084, - /* 2320 */ 1721, 671, 1721, 1721, 1721, 1721, 1721, 2134, 1721, 1721, - /* 2330 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 2084, - /* 2340 */ 1721, 671, 2134, 1721, 1721, 1721, 1721, 672, 1721, 1721, - /* 2350 */ 1721, 1721, 2115, 1721, 2084, 2151, 671, 1721, 324, 2117, - /* 2360 */ 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, 1721, - /* 2370 */ 1721, 1721, 2115, 1721, 2134, 2151, 1721, 1721, 333, 2117, - /* 2380 */ 675, 2119, 2120, 670, 1721, 665, 2084, 2115, 671, 1721, - /* 2390 */ 2151, 1721, 1721, 325, 2117, 675, 2119, 2120, 670, 1721, - /* 2400 */ 665, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, - /* 2410 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 2115, - /* 2420 */ 1721, 1721, 2151, 1721, 2116, 338, 2117, 675, 2119, 2120, - /* 2430 */ 670, 1721, 665, 1721, 672, 1721, 1721, 1721, 1721, 1721, - /* 2440 */ 1721, 1721, 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2450 */ 1721, 1721, 2116, 1721, 2084, 1721, 671, 1721, 1721, 1721, - /* 2460 */ 1721, 2134, 672, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2470 */ 1721, 2116, 1721, 2084, 1721, 671, 1721, 1721, 1721, 1721, - /* 2480 */ 1721, 672, 1721, 1721, 1721, 1721, 1721, 2115, 1721, 2134, - /* 2490 */ 2151, 1721, 1721, 339, 2117, 675, 2119, 2120, 670, 1721, - /* 2500 */ 665, 2084, 1721, 671, 1721, 1721, 2115, 1721, 2134, 2151, - /* 2510 */ 1721, 1721, 2128, 2117, 675, 2119, 2120, 670, 2116, 665, - /* 2520 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 672, 1721, - /* 2530 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 2151, 1721, 1721, - /* 2540 */ 2127, 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, - /* 2550 */ 1721, 1721, 1721, 2115, 1721, 2134, 2151, 1721, 1721, 2126, - /* 2560 */ 2117, 675, 2119, 2120, 670, 1721, 665, 2084, 1721, 671, - /* 2570 */ 1721, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, - /* 2580 */ 1721, 1721, 1721, 1721, 1721, 672, 1721, 1721, 1721, 1721, - /* 2590 */ 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2600 */ 2115, 672, 1721, 2151, 1721, 1721, 353, 2117, 675, 2119, - /* 2610 */ 2120, 670, 2134, 665, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2620 */ 1721, 1721, 1721, 1721, 2084, 1721, 671, 1721, 2134, 1721, - /* 2630 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2640 */ 2084, 2116, 671, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2650 */ 1721, 672, 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, - /* 2660 */ 2151, 2116, 1721, 354, 2117, 675, 2119, 2120, 670, 1721, - /* 2670 */ 665, 672, 1721, 2115, 1721, 1721, 2151, 1721, 2134, 350, - /* 2680 */ 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, - /* 2690 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 2134, 1721, - /* 2700 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, - /* 2710 */ 2084, 1721, 671, 1721, 1721, 1721, 1721, 1721, 672, 1721, - /* 2720 */ 1721, 1721, 1721, 2115, 1721, 1721, 2151, 1721, 1721, 355, - /* 2730 */ 2117, 675, 2119, 2120, 670, 1721, 665, 1721, 1721, 1721, - /* 2740 */ 1721, 1721, 229, 673, 1721, 2134, 2151, 1721, 1721, 330, - /* 2750 */ 2117, 675, 2119, 2120, 670, 1721, 665, 2084, 172, 671, - /* 2760 */ 1721, 1721, 1721, 1721, 535, 531, 527, 523, 226, 1721, - /* 2770 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2780 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2790 */ 2115, 1721, 1721, 2151, 1721, 1721, 329, 2117, 675, 2119, - /* 2800 */ 2120, 670, 1721, 665, 1721, 1721, 1721, 1721, 88, 1721, - /* 2810 */ 1721, 224, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2820 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2830 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2840 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2850 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2860 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2870 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 223, 217, - /* 2880 */ 1721, 1721, 1721, 222, 1721, 514, 1721, 1721, 1721, 1721, - /* 2890 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2900 */ 1721, 1721, 1721, 215, + /* 0 */ 503, 2118, 2282, 504, 1760, 2277, 36, 300, 1883, 442, + /* 10 */ 66, 674, 46, 44, 1648, 45, 43, 42, 41, 40, + /* 20 */ 395, 2281, 1497, 39, 38, 2278, 2280, 45, 43, 42, + /* 30 */ 41, 40, 1976, 1578, 1793, 1495, 657, 1894, 2136, 39, + /* 40 */ 38, 632, 140, 45, 43, 42, 41, 40, 657, 1894, + /* 50 */ 2086, 1724, 673, 342, 2012, 191, 39, 38, 573, 1573, + /* 60 */ 45, 43, 42, 41, 40, 19, 388, 133, 520, 2009, + /* 70 */ 644, 571, 1503, 569, 541, 132, 131, 130, 129, 128, + /* 80 */ 127, 126, 125, 124, 2117, 168, 2153, 1736, 198, 170, + /* 90 */ 2119, 677, 2121, 2122, 672, 511, 667, 760, 504, 1760, + /* 100 */ 15, 737, 736, 735, 734, 407, 62, 733, 732, 144, + /* 110 */ 727, 726, 725, 724, 723, 722, 721, 157, 717, 716, + /* 120 */ 715, 406, 405, 712, 711, 710, 175, 174, 84, 404, + /* 130 */ 403, 83, 1687, 656, 709, 1722, 1580, 1581, 1946, 1511, + /* 140 */ 622, 2298, 278, 2214, 631, 375, 134, 630, 1522, 2277, + /* 150 */ 1578, 123, 1504, 1944, 122, 121, 120, 119, 118, 117, + /* 160 */ 116, 115, 114, 2012, 619, 186, 1553, 1563, 1946, 2278, + /* 170 */ 621, 2282, 1579, 1582, 2277, 360, 1573, 400, 2010, 644, + /* 180 */ 1939, 1941, 656, 1944, 398, 1522, 1498, 460, 1496, 1503, + /* 190 */ 2281, 87, 162, 1717, 2278, 2279, 459, 39, 38, 376, + /* 200 */ 1896, 45, 43, 42, 41, 40, 410, 1944, 364, 582, + /* 210 */ 1522, 409, 62, 49, 663, 1501, 1502, 1889, 1552, 1555, + /* 220 */ 1556, 1557, 1558, 1559, 1560, 1561, 1562, 669, 665, 1571, + /* 230 */ 1572, 1574, 1575, 1576, 1577, 2, 46, 44, 632, 140, + /* 240 */ 1171, 345, 1523, 1520, 395, 620, 1497, 480, 2277, 12, + /* 250 */ 473, 10, 595, 487, 354, 2277, 486, 1578, 189, 1495, + /* 260 */ 443, 508, 229, 619, 186, 62, 2096, 505, 2278, 621, + /* 270 */ 2283, 186, 456, 444, 488, 2278, 621, 458, 172, 1173, + /* 280 */ 2104, 1176, 1177, 1573, 537, 533, 529, 525, 226, 19, + /* 290 */ 2100, 1523, 1716, 559, 558, 557, 1503, 1193, 1607, 1192, + /* 300 */ 549, 137, 553, 1512, 1747, 1507, 552, 202, 201, 14, + /* 310 */ 13, 551, 556, 370, 369, 50, 642, 550, 2282, 404, + /* 320 */ 403, 760, 363, 615, 15, 182, 2102, 378, 88, 1194, + /* 330 */ 479, 224, 1515, 1517, 446, 1524, 667, 1933, 634, 184, + /* 340 */ 2214, 2215, 1504, 138, 2219, 665, 1571, 1572, 1574, 1575, + /* 350 */ 1576, 1577, 664, 2086, 1608, 1746, 1411, 1412, 1428, 1429, + /* 360 */ 1580, 1581, 179, 484, 189, 189, 478, 477, 476, 475, + /* 370 */ 472, 471, 470, 469, 468, 464, 463, 462, 461, 344, + /* 380 */ 453, 452, 451, 1996, 448, 447, 361, 657, 1894, 1277, + /* 390 */ 1553, 1563, 657, 1894, 1427, 1430, 1579, 1582, 223, 217, + /* 400 */ 2221, 610, 1276, 222, 2086, 516, 133, 1356, 1357, 1621, + /* 410 */ 1498, 55, 1496, 546, 39, 38, 1710, 189, 45, 43, + /* 420 */ 42, 41, 40, 215, 1796, 1871, 2218, 1676, 35, 393, + /* 430 */ 1602, 1603, 1604, 1605, 1606, 1610, 1611, 1612, 1613, 1501, + /* 440 */ 1502, 1745, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, + /* 450 */ 1562, 669, 665, 1571, 1572, 1574, 1575, 1576, 1577, 2, + /* 460 */ 12, 46, 44, 2107, 489, 167, 1652, 189, 1497, 395, + /* 470 */ 367, 1497, 1522, 1835, 607, 606, 1674, 1675, 1677, 1678, + /* 480 */ 1679, 1495, 1578, 280, 1495, 179, 2118, 616, 611, 604, + /* 490 */ 2086, 559, 558, 557, 643, 1507, 671, 1588, 549, 137, + /* 500 */ 553, 1725, 1599, 1522, 552, 362, 1995, 1664, 1573, 551, + /* 510 */ 556, 370, 369, 211, 19, 550, 2109, 506, 1503, 1767, + /* 520 */ 1870, 1503, 123, 2136, 2053, 122, 121, 120, 119, 118, + /* 530 */ 117, 116, 115, 114, 614, 2086, 368, 673, 366, 365, + /* 540 */ 429, 543, 1524, 760, 213, 518, 760, 2005, 506, 15, + /* 550 */ 1767, 657, 1894, 2118, 707, 155, 154, 704, 703, 702, + /* 560 */ 152, 2136, 545, 674, 431, 427, 544, 656, 166, 2117, + /* 570 */ 247, 2153, 246, 320, 336, 2119, 677, 2121, 2122, 672, + /* 580 */ 670, 667, 658, 2171, 709, 1580, 1581, 318, 73, 2281, + /* 590 */ 2136, 72, 39, 38, 1940, 1941, 45, 43, 42, 41, + /* 600 */ 40, 389, 2086, 1521, 673, 101, 42, 41, 40, 165, + /* 610 */ 209, 499, 497, 494, 613, 1553, 1563, 1896, 1554, 39, + /* 620 */ 38, 1579, 1582, 45, 43, 42, 41, 40, 28, 1266, + /* 630 */ 1887, 1525, 1498, 12, 1496, 1498, 2117, 1496, 2153, 657, + /* 640 */ 1894, 110, 2119, 677, 2121, 2122, 672, 49, 667, 1554, + /* 650 */ 62, 143, 436, 150, 2177, 2206, 700, 435, 440, 391, + /* 660 */ 2202, 1501, 1502, 1879, 1501, 1502, 1268, 1552, 1555, 1556, + /* 670 */ 1557, 1558, 1559, 1560, 1561, 1562, 669, 665, 1571, 1572, + /* 680 */ 1574, 1575, 1576, 1577, 2, 46, 44, 1583, 109, 1946, + /* 690 */ 280, 643, 1281, 395, 87, 1497, 385, 2118, 595, 142, + /* 700 */ 620, 2277, 2177, 2277, 1944, 1280, 1578, 635, 1495, 707, + /* 710 */ 155, 154, 704, 703, 702, 152, 2283, 186, 619, 186, + /* 720 */ 1890, 2278, 621, 2278, 621, 282, 555, 554, 81, 80, + /* 730 */ 439, 1503, 1573, 193, 2136, 2096, 1946, 1193, 659, 1192, + /* 740 */ 2178, 2096, 641, 390, 2005, 1503, 2086, 701, 673, 2104, + /* 750 */ 1937, 1944, 228, 1991, 343, 1885, 1946, 425, 2221, 2100, + /* 760 */ 423, 419, 415, 412, 432, 2100, 62, 491, 93, 1194, + /* 770 */ 760, 1945, 643, 47, 2221, 398, 291, 292, 1744, 258, + /* 780 */ 2117, 290, 2153, 165, 2217, 110, 2119, 677, 2121, 2122, + /* 790 */ 672, 1896, 667, 1869, 1223, 2102, 379, 183, 194, 2206, + /* 800 */ 2216, 2102, 189, 391, 2202, 667, 1743, 32, 1742, 1580, + /* 810 */ 1581, 667, 8, 39, 38, 1741, 188, 45, 43, 42, + /* 820 */ 41, 40, 34, 652, 2232, 2005, 545, 2086, 39, 38, + /* 830 */ 544, 1224, 45, 43, 42, 41, 40, 657, 1894, 1553, + /* 840 */ 1563, 657, 1894, 39, 38, 1579, 1582, 45, 43, 42, + /* 850 */ 41, 40, 1322, 731, 729, 2086, 519, 2086, 623, 1498, + /* 860 */ 441, 1496, 2226, 1641, 2086, 657, 1894, 1313, 699, 698, + /* 870 */ 697, 1317, 696, 1319, 1320, 695, 692, 1609, 1328, 689, + /* 880 */ 1330, 1331, 686, 683, 450, 2069, 632, 140, 1501, 1502, + /* 890 */ 1740, 1552, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, + /* 900 */ 669, 665, 1571, 1572, 1574, 1575, 1576, 1577, 2, 46, + /* 910 */ 44, 245, 107, 1176, 1177, 62, 638, 395, 189, 1497, + /* 920 */ 719, 2118, 707, 155, 154, 704, 703, 702, 152, 141, + /* 930 */ 1578, 635, 1495, 657, 1894, 657, 1894, 1886, 661, 2086, + /* 940 */ 2178, 657, 1894, 1179, 1881, 657, 1894, 2118, 1946, 1521, + /* 950 */ 1739, 33, 465, 1645, 466, 399, 1573, 674, 2136, 1769, + /* 960 */ 1891, 1614, 595, 1944, 255, 2277, 1525, 1877, 165, 1503, + /* 970 */ 2086, 1738, 673, 1735, 632, 140, 1897, 1734, 1470, 1471, + /* 980 */ 2283, 186, 1733, 627, 2136, 2278, 621, 185, 2214, 2215, + /* 990 */ 314, 138, 2219, 1923, 760, 1898, 2086, 47, 673, 2086, + /* 1000 */ 434, 2096, 433, 164, 2117, 1991, 2153, 2118, 1732, 110, + /* 1010 */ 2119, 677, 2121, 2122, 672, 2104, 667, 674, 1525, 1731, + /* 1020 */ 2086, 183, 2086, 2206, 1641, 2100, 2086, 391, 2202, 432, + /* 1030 */ 2117, 2086, 2153, 1580, 1581, 110, 2119, 677, 2121, 2122, + /* 1040 */ 672, 244, 667, 401, 2136, 243, 1522, 2297, 2233, 2206, + /* 1050 */ 196, 165, 1730, 391, 2202, 1729, 2086, 2086, 673, 1896, + /* 1060 */ 547, 2102, 392, 1553, 1563, 657, 1894, 189, 2086, 1579, + /* 1070 */ 1582, 667, 657, 1894, 624, 187, 2214, 2215, 705, 138, + /* 1080 */ 2219, 1937, 1264, 1498, 281, 1496, 1719, 1720, 657, 1894, + /* 1090 */ 2117, 640, 2153, 1728, 424, 169, 2119, 677, 2121, 2122, + /* 1100 */ 672, 2086, 667, 251, 2086, 90, 349, 295, 1991, 374, + /* 1110 */ 668, 575, 1501, 1502, 195, 1552, 1555, 1556, 1557, 1558, + /* 1120 */ 1559, 1560, 1561, 1562, 669, 665, 1571, 1572, 1574, 1575, + /* 1130 */ 1576, 1577, 2, 46, 44, 596, 2243, 2118, 2096, 657, + /* 1140 */ 1894, 395, 2086, 1497, 657, 1894, 2072, 674, 74, 2253, + /* 1150 */ 657, 1894, 2105, 200, 1578, 1727, 1495, 153, 654, 657, + /* 1160 */ 1894, 564, 2100, 655, 706, 1872, 257, 1937, 720, 301, + /* 1170 */ 52, 1856, 3, 146, 2136, 135, 574, 577, 402, 576, + /* 1180 */ 1573, 234, 236, 548, 232, 235, 2086, 1836, 673, 153, + /* 1190 */ 242, 1783, 1554, 1503, 153, 417, 64, 82, 2102, 2079, + /* 1200 */ 238, 2080, 1644, 237, 2086, 1262, 567, 240, 667, 148, + /* 1210 */ 239, 561, 1776, 560, 54, 628, 241, 1506, 760, 1774, + /* 1220 */ 2117, 15, 2153, 2118, 64, 110, 2119, 677, 2121, 2122, + /* 1230 */ 672, 580, 667, 674, 562, 602, 256, 2297, 594, 2206, + /* 1240 */ 262, 565, 153, 391, 2202, 595, 1465, 595, 2277, 1505, + /* 1250 */ 2277, 1468, 1770, 1673, 254, 1737, 70, 1580, 1581, 69, + /* 1260 */ 2136, 14, 13, 2283, 186, 2283, 186, 48, 2278, 621, + /* 1270 */ 2278, 621, 2086, 106, 673, 288, 71, 595, 151, 91, + /* 1280 */ 2277, 1672, 608, 103, 2246, 1834, 275, 1553, 1563, 227, + /* 1290 */ 269, 1833, 2137, 1579, 1582, 2283, 186, 264, 408, 639, + /* 1300 */ 2278, 621, 755, 153, 625, 64, 2117, 1498, 2153, 1496, + /* 1310 */ 2000, 110, 2119, 677, 2121, 2122, 672, 48, 667, 48, + /* 1320 */ 681, 151, 153, 2297, 1425, 2206, 1761, 1766, 1934, 391, + /* 1330 */ 2202, 633, 293, 649, 2236, 297, 1501, 1502, 277, 1552, + /* 1340 */ 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, 669, 665, + /* 1350 */ 1571, 1572, 1574, 1575, 1576, 1577, 2, 2118, 53, 136, + /* 1360 */ 1307, 274, 1615, 713, 151, 1, 714, 674, 9, 2271, + /* 1370 */ 1509, 411, 416, 358, 1564, 308, 313, 1334, 1338, 1345, + /* 1380 */ 199, 1448, 445, 2118, 1525, 1242, 2001, 482, 1240, 449, + /* 1390 */ 454, 1520, 481, 674, 2136, 2225, 467, 1993, 474, 483, + /* 1400 */ 492, 490, 1508, 493, 203, 204, 2086, 495, 673, 496, + /* 1410 */ 206, 498, 500, 1526, 501, 4, 1343, 509, 1528, 502, + /* 1420 */ 2136, 156, 510, 214, 512, 1523, 513, 1527, 514, 1529, + /* 1430 */ 515, 216, 2086, 517, 673, 219, 221, 85, 86, 521, + /* 1440 */ 2117, 225, 2153, 1196, 540, 110, 2119, 677, 2121, 2122, + /* 1450 */ 672, 538, 667, 539, 2062, 2118, 2059, 2297, 2058, 2206, + /* 1460 */ 542, 112, 1884, 391, 2202, 674, 2117, 2240, 2153, 231, + /* 1470 */ 1880, 110, 2119, 677, 2121, 2122, 672, 233, 667, 348, + /* 1480 */ 158, 2118, 159, 2297, 579, 2206, 1882, 1878, 160, 391, + /* 1490 */ 2202, 674, 2136, 161, 89, 248, 585, 584, 1455, 589, + /* 1500 */ 592, 252, 609, 2237, 2086, 586, 673, 149, 581, 309, + /* 1510 */ 250, 599, 2252, 590, 2118, 647, 2247, 605, 2136, 380, + /* 1520 */ 2251, 612, 591, 260, 674, 7, 2228, 263, 618, 268, + /* 1530 */ 2086, 173, 673, 600, 598, 597, 273, 2300, 2117, 626, + /* 1540 */ 2153, 381, 629, 110, 2119, 677, 2121, 2122, 672, 1641, + /* 1550 */ 667, 2136, 139, 1524, 384, 2297, 271, 2206, 2222, 637, + /* 1560 */ 1530, 391, 2202, 2086, 2117, 673, 2153, 2006, 636, 110, + /* 1570 */ 2119, 677, 2121, 2122, 672, 283, 667, 2276, 96, 312, + /* 1580 */ 270, 2181, 310, 2206, 645, 272, 646, 391, 2202, 2020, + /* 1590 */ 2019, 2118, 311, 2018, 650, 276, 98, 2117, 387, 2153, + /* 1600 */ 651, 674, 110, 2119, 677, 2121, 2122, 672, 61, 667, + /* 1610 */ 102, 1895, 2187, 100, 2179, 1938, 2206, 679, 304, 756, + /* 1620 */ 391, 2202, 2118, 757, 315, 759, 51, 339, 2136, 1857, + /* 1630 */ 319, 2078, 674, 350, 351, 324, 338, 317, 328, 2077, + /* 1640 */ 2086, 2076, 673, 78, 2073, 413, 414, 1488, 1489, 192, + /* 1650 */ 2118, 418, 2071, 420, 421, 422, 2070, 359, 2068, 2136, + /* 1660 */ 674, 426, 2067, 2066, 428, 430, 79, 1451, 1450, 2032, + /* 1670 */ 2031, 2086, 2030, 673, 2117, 437, 2153, 438, 2029, 110, + /* 1680 */ 2119, 677, 2121, 2122, 672, 2118, 667, 2136, 2028, 1984, + /* 1690 */ 1402, 660, 1983, 2206, 1981, 674, 145, 391, 2202, 2086, + /* 1700 */ 1980, 673, 1979, 1982, 1978, 2117, 1977, 2153, 1975, 1974, + /* 1710 */ 111, 2119, 677, 2121, 2122, 672, 1973, 667, 455, 197, + /* 1720 */ 1972, 457, 2136, 1986, 2206, 1971, 1970, 1969, 2205, 2202, + /* 1730 */ 1968, 1967, 1966, 2117, 2086, 2153, 673, 1965, 111, 2119, + /* 1740 */ 677, 2121, 2122, 672, 1964, 667, 1963, 1962, 1961, 1960, + /* 1750 */ 1959, 1958, 2206, 1957, 147, 1956, 662, 2202, 1955, 1954, + /* 1760 */ 1985, 1953, 1952, 1404, 1951, 1950, 1949, 485, 675, 346, + /* 1770 */ 2153, 1948, 1947, 111, 2119, 677, 2121, 2122, 672, 2118, + /* 1780 */ 667, 347, 1799, 205, 1798, 1278, 1797, 2206, 207, 674, + /* 1790 */ 1282, 353, 2202, 1795, 208, 1756, 210, 1178, 1755, 2118, + /* 1800 */ 180, 2049, 1274, 2039, 2106, 2027, 220, 2026, 2004, 674, + /* 1810 */ 2118, 1873, 76, 1794, 1792, 212, 2136, 181, 77, 218, + /* 1820 */ 674, 507, 522, 1790, 524, 526, 523, 527, 2086, 1788, + /* 1830 */ 673, 528, 530, 532, 1786, 534, 2136, 531, 535, 1773, + /* 1840 */ 1772, 536, 1216, 1752, 1875, 63, 230, 2136, 2086, 1349, + /* 1850 */ 673, 1350, 1874, 1265, 1263, 1261, 1260, 1259, 1252, 2086, + /* 1860 */ 1258, 673, 2117, 1257, 2153, 1254, 1784, 111, 2119, 677, + /* 1870 */ 2121, 2122, 672, 1253, 667, 371, 728, 730, 1777, 372, + /* 1880 */ 1251, 2206, 2117, 1775, 2153, 373, 2203, 169, 2119, 677, + /* 1890 */ 2121, 2122, 672, 2117, 667, 2153, 2118, 563, 330, 2119, + /* 1900 */ 677, 2121, 2122, 672, 566, 667, 674, 2118, 1751, 568, + /* 1910 */ 1750, 570, 1749, 572, 113, 1475, 1477, 674, 1479, 1474, + /* 1920 */ 2048, 1461, 2118, 1459, 27, 1457, 67, 2038, 2244, 163, + /* 1930 */ 2025, 587, 671, 2136, 2023, 2282, 20, 17, 1689, 601, + /* 1940 */ 59, 617, 29, 5, 2136, 2086, 6, 673, 259, 386, + /* 1950 */ 603, 253, 261, 267, 2107, 60, 2086, 1671, 673, 2136, + /* 1960 */ 171, 265, 30, 266, 56, 1663, 593, 588, 92, 31, + /* 1970 */ 21, 2086, 377, 673, 1709, 65, 1710, 22, 18, 2117, + /* 1980 */ 1704, 2153, 1703, 382, 170, 2119, 677, 2121, 2122, 672, + /* 1990 */ 2117, 667, 2153, 2118, 1708, 337, 2119, 677, 2121, 2122, + /* 2000 */ 672, 1707, 667, 674, 383, 2117, 1638, 2153, 2118, 279, + /* 2010 */ 336, 2119, 677, 2121, 2122, 672, 583, 667, 674, 2172, + /* 2020 */ 58, 176, 1637, 2024, 2022, 2021, 2003, 95, 94, 286, + /* 2030 */ 2136, 23, 2002, 97, 763, 394, 2299, 287, 1669, 289, + /* 2040 */ 294, 648, 2086, 68, 673, 2136, 296, 299, 307, 99, + /* 2050 */ 396, 103, 24, 1590, 1589, 13, 57, 2086, 1513, 673, + /* 2060 */ 2156, 666, 1600, 37, 178, 1568, 1566, 2118, 1565, 177, + /* 2070 */ 753, 749, 745, 741, 305, 16, 2117, 674, 2153, 25, + /* 2080 */ 190, 337, 2119, 677, 2121, 2122, 672, 11, 667, 1545, + /* 2090 */ 1537, 2117, 26, 2153, 678, 680, 337, 2119, 677, 2121, + /* 2100 */ 2122, 672, 2118, 667, 2136, 397, 676, 1335, 1332, 682, + /* 2110 */ 684, 685, 674, 1329, 108, 687, 2086, 298, 673, 688, + /* 2120 */ 690, 693, 1323, 1327, 1326, 1325, 302, 691, 1344, 1321, + /* 2130 */ 694, 104, 1324, 1340, 105, 1214, 708, 75, 1246, 2136, + /* 2140 */ 1245, 1244, 1243, 1241, 1239, 1238, 1272, 1237, 718, 653, + /* 2150 */ 578, 2086, 2153, 673, 1232, 332, 2119, 677, 2121, 2122, + /* 2160 */ 672, 303, 667, 1235, 2118, 1234, 1233, 1269, 1231, 1230, + /* 2170 */ 1229, 1267, 1226, 1225, 674, 1222, 1221, 1220, 1219, 1791, + /* 2180 */ 738, 740, 1789, 742, 285, 2117, 1787, 2153, 739, 284, + /* 2190 */ 321, 2119, 677, 2121, 2122, 672, 743, 667, 744, 746, + /* 2200 */ 747, 2136, 748, 1785, 750, 751, 752, 1771, 754, 249, + /* 2210 */ 1168, 1748, 306, 2086, 758, 673, 1499, 316, 761, 762, + /* 2220 */ 1723, 2118, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2230 */ 1723, 674, 2118, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2240 */ 1723, 1723, 674, 1723, 1723, 1723, 1723, 2117, 1723, 2153, + /* 2250 */ 1723, 2118, 322, 2119, 677, 2121, 2122, 672, 2136, 667, + /* 2260 */ 1723, 674, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2136, + /* 2270 */ 2086, 1723, 673, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2280 */ 1723, 2086, 1723, 673, 1723, 1723, 1723, 1723, 2136, 1723, + /* 2290 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2300 */ 2086, 1723, 673, 1723, 2117, 1723, 2153, 1723, 1723, 323, + /* 2310 */ 2119, 677, 2121, 2122, 672, 2117, 667, 2153, 2118, 1723, + /* 2320 */ 329, 2119, 677, 2121, 2122, 672, 1723, 667, 674, 1723, + /* 2330 */ 1723, 1723, 1723, 1723, 2117, 1723, 2153, 2118, 1723, 333, + /* 2340 */ 2119, 677, 2121, 2122, 672, 1723, 667, 674, 1723, 1723, + /* 2350 */ 1723, 1723, 1723, 2118, 1723, 2136, 1723, 1723, 1723, 1723, + /* 2360 */ 1723, 1723, 1723, 674, 1723, 1723, 1723, 2086, 2118, 673, + /* 2370 */ 1723, 1723, 1723, 1723, 2136, 1723, 1723, 1723, 674, 1723, + /* 2380 */ 1723, 1723, 1723, 1723, 2118, 1723, 2086, 1723, 673, 1723, + /* 2390 */ 2136, 1723, 1723, 1723, 674, 1723, 1723, 1723, 1723, 1723, + /* 2400 */ 1723, 2117, 2086, 2153, 673, 2136, 325, 2119, 677, 2121, + /* 2410 */ 2122, 672, 1723, 667, 1723, 1723, 1723, 2086, 1723, 673, + /* 2420 */ 2117, 2136, 2153, 1723, 1723, 334, 2119, 677, 2121, 2122, + /* 2430 */ 672, 1723, 667, 2086, 1723, 673, 2117, 1723, 2153, 1723, + /* 2440 */ 1723, 326, 2119, 677, 2121, 2122, 672, 1723, 667, 1723, + /* 2450 */ 1723, 2117, 2118, 2153, 1723, 1723, 335, 2119, 677, 2121, + /* 2460 */ 2122, 672, 674, 667, 1723, 1723, 1723, 2117, 1723, 2153, + /* 2470 */ 1723, 1723, 327, 2119, 677, 2121, 2122, 672, 1723, 667, + /* 2480 */ 1723, 1723, 1723, 2118, 1723, 1723, 1723, 1723, 1723, 2136, + /* 2490 */ 1723, 1723, 1723, 674, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2500 */ 1723, 2086, 1723, 673, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2510 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2118, 1723, + /* 2520 */ 2136, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 674, 1723, + /* 2530 */ 1723, 1723, 2086, 1723, 673, 2117, 1723, 2153, 1723, 1723, + /* 2540 */ 340, 2119, 677, 2121, 2122, 672, 1723, 667, 1723, 2118, + /* 2550 */ 1723, 1723, 1723, 1723, 1723, 2136, 1723, 1723, 1723, 674, + /* 2560 */ 1723, 1723, 1723, 1723, 1723, 1723, 2117, 2086, 2153, 673, + /* 2570 */ 1723, 341, 2119, 677, 2121, 2122, 672, 1723, 667, 1723, + /* 2580 */ 1723, 1723, 1723, 1723, 1723, 1723, 2136, 1723, 1723, 1723, + /* 2590 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2086, 1723, + /* 2600 */ 673, 2117, 1723, 2153, 1723, 1723, 2130, 2119, 677, 2121, + /* 2610 */ 2122, 672, 1723, 667, 1723, 2118, 1723, 1723, 1723, 1723, + /* 2620 */ 1723, 1723, 1723, 1723, 1723, 674, 1723, 1723, 1723, 1723, + /* 2630 */ 1723, 1723, 2117, 1723, 2153, 1723, 2118, 2129, 2119, 677, + /* 2640 */ 2121, 2122, 672, 1723, 667, 1723, 674, 1723, 1723, 1723, + /* 2650 */ 1723, 1723, 2136, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2660 */ 1723, 1723, 2118, 1723, 2086, 1723, 673, 1723, 1723, 1723, + /* 2670 */ 1723, 1723, 674, 2136, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2680 */ 1723, 2118, 1723, 1723, 1723, 2086, 1723, 673, 1723, 1723, + /* 2690 */ 1723, 674, 1723, 1723, 1723, 1723, 1723, 1723, 2117, 2136, + /* 2700 */ 2153, 1723, 1723, 2128, 2119, 677, 2121, 2122, 672, 1723, + /* 2710 */ 667, 2086, 2118, 673, 1723, 1723, 1723, 1723, 2136, 2117, + /* 2720 */ 1723, 2153, 674, 1723, 355, 2119, 677, 2121, 2122, 672, + /* 2730 */ 2086, 667, 673, 1723, 1723, 1723, 1723, 1723, 2118, 1723, + /* 2740 */ 1723, 1723, 1723, 1723, 1723, 2117, 1723, 2153, 674, 2136, + /* 2750 */ 356, 2119, 677, 2121, 2122, 672, 1723, 667, 1723, 1723, + /* 2760 */ 1723, 2086, 1723, 673, 2117, 1723, 2153, 1723, 1723, 352, + /* 2770 */ 2119, 677, 2121, 2122, 672, 2136, 667, 1723, 1723, 1723, + /* 2780 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2086, 1723, 673, + /* 2790 */ 1723, 1723, 1723, 1723, 1723, 2117, 1723, 2153, 1723, 1723, + /* 2800 */ 357, 2119, 677, 2121, 2122, 672, 2118, 667, 1723, 1723, + /* 2810 */ 1723, 1723, 1723, 1723, 1723, 1723, 674, 1723, 1723, 1723, + /* 2820 */ 1723, 675, 1723, 2153, 1723, 1723, 332, 2119, 677, 2121, + /* 2830 */ 2122, 672, 1723, 667, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2840 */ 1723, 1723, 1723, 2136, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2850 */ 1723, 1723, 1723, 1723, 1723, 2086, 1723, 673, 1723, 1723, + /* 2860 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2870 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, + /* 2880 */ 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 1723, 2117, + /* 2890 */ 1723, 2153, 1723, 1723, 331, 2119, 677, 2121, 2122, 672, + /* 2900 */ 1723, 667, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 333, 384, 371, 362, 401, 332, 401, 334, 405, 343, - /* 10 */ 343, 370, 12, 13, 14, 0, 399, 400, 377, 378, - /* 20 */ 20, 14, 22, 342, 8, 9, 385, 20, 12, 13, - /* 30 */ 14, 15, 16, 33, 0, 35, 21, 370, 20, 24, - /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 3, 382, - /* 50 */ 447, 384, 447, 450, 388, 450, 8, 9, 351, 59, - /* 60 */ 12, 13, 14, 15, 16, 65, 359, 386, 465, 466, - /* 70 */ 465, 466, 72, 470, 471, 470, 471, 12, 13, 14, - /* 80 */ 15, 16, 415, 337, 20, 418, 340, 341, 421, 422, - /* 90 */ 423, 424, 425, 426, 369, 428, 20, 97, 436, 437, - /* 100 */ 100, 67, 68, 69, 70, 71, 381, 73, 74, 75, + /* 0 */ 337, 333, 447, 340, 341, 450, 436, 437, 371, 342, + /* 10 */ 4, 343, 12, 13, 14, 12, 13, 14, 15, 16, + /* 20 */ 20, 466, 22, 8, 9, 470, 471, 12, 13, 14, + /* 30 */ 15, 16, 0, 33, 0, 35, 342, 343, 370, 8, + /* 40 */ 9, 342, 343, 12, 13, 14, 15, 16, 342, 343, + /* 50 */ 382, 0, 384, 386, 384, 361, 8, 9, 21, 59, + /* 60 */ 12, 13, 14, 15, 16, 65, 396, 361, 64, 399, + /* 70 */ 400, 34, 72, 36, 368, 24, 25, 26, 27, 28, + /* 80 */ 29, 30, 31, 32, 416, 332, 418, 334, 59, 421, + /* 90 */ 422, 423, 424, 425, 426, 337, 428, 97, 340, 341, + /* 100 */ 100, 67, 68, 69, 70, 71, 100, 73, 74, 75, /* 110 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, - /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 461, 462, - /* 130 */ 337, 20, 97, 340, 341, 21, 136, 137, 24, 25, - /* 140 */ 26, 27, 28, 29, 30, 31, 32, 112, 113, 114, - /* 150 */ 115, 116, 117, 118, 119, 120, 121, 370, 123, 124, - /* 160 */ 125, 126, 127, 128, 100, 20, 166, 167, 20, 420, - /* 170 */ 342, 343, 172, 173, 330, 8, 9, 390, 391, 12, - /* 180 */ 13, 14, 15, 16, 166, 167, 186, 20, 188, 361, - /* 190 */ 342, 343, 362, 100, 178, 446, 368, 8, 9, 59, - /* 200 */ 370, 12, 13, 14, 15, 16, 8, 9, 378, 361, - /* 210 */ 12, 13, 14, 15, 16, 215, 216, 0, 218, 219, + /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 99, 12, + /* 130 */ 13, 102, 101, 20, 64, 330, 136, 137, 370, 22, + /* 140 */ 472, 473, 443, 444, 445, 377, 447, 448, 20, 450, + /* 150 */ 33, 21, 35, 385, 24, 25, 26, 27, 28, 29, + /* 160 */ 30, 31, 32, 384, 465, 466, 166, 167, 370, 470, + /* 170 */ 471, 447, 172, 173, 450, 377, 59, 380, 399, 400, + /* 180 */ 383, 384, 20, 385, 362, 20, 186, 155, 188, 72, + /* 190 */ 466, 350, 370, 178, 470, 471, 164, 8, 9, 377, + /* 200 */ 378, 12, 13, 14, 15, 16, 401, 385, 367, 111, + /* 210 */ 20, 406, 100, 100, 97, 215, 216, 376, 218, 219, /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 0, 370, - /* 240 */ 100, 18, 102, 20, 20, 401, 22, 100, 100, 405, - /* 250 */ 27, 333, 447, 30, 65, 450, 33, 33, 343, 35, - /* 260 */ 391, 343, 24, 25, 26, 27, 28, 29, 30, 31, - /* 270 */ 32, 466, 49, 20, 51, 470, 471, 54, 185, 362, - /* 280 */ 187, 64, 447, 59, 22, 450, 338, 370, 370, 65, - /* 290 */ 342, 447, 344, 277, 450, 378, 72, 35, 109, 101, - /* 300 */ 382, 466, 384, 388, 384, 470, 471, 214, 447, 465, - /* 310 */ 466, 450, 64, 333, 470, 471, 396, 272, 4, 399, - /* 320 */ 400, 97, 99, 343, 100, 345, 465, 466, 35, 342, - /* 330 */ 343, 470, 471, 415, 111, 168, 418, 342, 343, 421, - /* 340 */ 422, 423, 424, 425, 426, 252, 428, 20, 361, 431, - /* 350 */ 370, 433, 434, 435, 165, 368, 361, 439, 440, 97, - /* 360 */ 136, 137, 382, 140, 384, 72, 143, 144, 145, 146, + /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 342, 343, + /* 240 */ 4, 18, 20, 20, 20, 447, 22, 81, 450, 236, + /* 250 */ 27, 238, 447, 30, 65, 450, 33, 33, 252, 35, + /* 260 */ 22, 14, 33, 465, 466, 100, 358, 20, 470, 471, + /* 270 */ 465, 466, 49, 35, 51, 470, 471, 54, 49, 43, + /* 280 */ 372, 45, 46, 59, 55, 56, 57, 58, 59, 65, + /* 290 */ 382, 20, 277, 67, 68, 69, 72, 20, 109, 22, + /* 300 */ 74, 75, 76, 186, 333, 188, 80, 141, 142, 1, + /* 310 */ 2, 85, 86, 87, 88, 100, 20, 91, 3, 12, + /* 320 */ 13, 97, 99, 20, 100, 369, 418, 419, 99, 52, + /* 330 */ 164, 102, 215, 216, 111, 20, 428, 381, 442, 443, + /* 340 */ 444, 445, 35, 447, 448, 228, 229, 230, 231, 232, + /* 350 */ 233, 234, 65, 382, 165, 333, 166, 167, 136, 137, + /* 360 */ 136, 137, 370, 140, 252, 252, 143, 144, 145, 146, /* 370 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, - /* 380 */ 157, 158, 159, 20, 161, 162, 163, 14, 20, 20, - /* 390 */ 166, 167, 252, 20, 13, 415, 172, 173, 418, 252, - /* 400 */ 252, 421, 422, 423, 424, 425, 426, 64, 428, 20, - /* 410 */ 186, 22, 188, 433, 100, 435, 35, 342, 343, 439, - /* 420 */ 440, 168, 100, 171, 342, 342, 343, 333, 239, 240, + /* 380 */ 157, 158, 159, 391, 161, 162, 163, 342, 343, 22, + /* 390 */ 166, 167, 342, 343, 172, 173, 172, 173, 169, 170, + /* 400 */ 420, 171, 35, 174, 382, 176, 361, 136, 137, 101, + /* 410 */ 186, 361, 188, 368, 8, 9, 101, 252, 12, 13, + /* 420 */ 14, 15, 16, 194, 0, 0, 446, 215, 239, 240, /* 430 */ 241, 242, 243, 244, 245, 246, 247, 248, 249, 215, - /* 440 */ 216, 52, 218, 219, 220, 221, 222, 223, 224, 225, + /* 440 */ 216, 333, 218, 219, 220, 221, 222, 223, 224, 225, /* 450 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - /* 460 */ 236, 12, 13, 136, 137, 22, 12, 13, 100, 20, - /* 470 */ 22, 22, 67, 68, 69, 393, 382, 395, 35, 74, - /* 480 */ 75, 76, 33, 35, 35, 80, 3, 333, 2, 35, - /* 490 */ 85, 86, 87, 88, 8, 9, 91, 343, 12, 13, - /* 500 */ 14, 15, 16, 20, 20, 136, 137, 333, 59, 257, - /* 510 */ 258, 259, 215, 379, 65, 72, 382, 343, 443, 444, - /* 520 */ 445, 72, 447, 448, 370, 442, 443, 444, 445, 380, - /* 530 */ 447, 448, 383, 384, 366, 20, 382, 22, 384, 166, - /* 540 */ 97, 172, 173, 20, 370, 97, 97, 8, 9, 100, - /* 550 */ 35, 12, 13, 14, 15, 16, 382, 44, 384, 262, - /* 560 */ 263, 264, 265, 266, 267, 268, 252, 52, 18, 415, - /* 570 */ 333, 111, 418, 23, 252, 421, 422, 423, 424, 425, - /* 580 */ 426, 413, 428, 44, 101, 136, 137, 37, 38, 415, - /* 590 */ 236, 41, 418, 358, 4, 421, 422, 423, 424, 425, - /* 600 */ 426, 427, 428, 429, 430, 342, 343, 372, 348, 19, - /* 610 */ 60, 61, 62, 63, 101, 166, 167, 382, 343, 382, - /* 620 */ 252, 172, 173, 33, 361, 365, 472, 473, 338, 186, - /* 630 */ 81, 188, 342, 373, 344, 186, 0, 188, 370, 49, - /* 640 */ 370, 370, 188, 342, 54, 370, 378, 377, 377, 59, - /* 650 */ 100, 168, 168, 418, 419, 385, 385, 72, 215, 216, - /* 660 */ 4, 130, 131, 428, 215, 216, 135, 218, 219, 220, + /* 460 */ 236, 12, 13, 47, 97, 351, 14, 252, 22, 20, + /* 470 */ 37, 22, 20, 359, 262, 263, 264, 265, 266, 267, + /* 480 */ 268, 35, 33, 168, 35, 370, 333, 257, 258, 259, + /* 490 */ 382, 67, 68, 69, 342, 188, 343, 14, 74, 75, + /* 500 */ 76, 0, 215, 20, 80, 390, 391, 101, 59, 85, + /* 510 */ 86, 87, 88, 338, 65, 91, 100, 342, 72, 344, + /* 520 */ 0, 72, 21, 370, 366, 24, 25, 26, 27, 28, + /* 530 */ 29, 30, 31, 32, 343, 382, 103, 384, 105, 106, + /* 540 */ 181, 108, 20, 97, 338, 393, 97, 395, 342, 100, + /* 550 */ 344, 342, 343, 333, 129, 130, 131, 132, 133, 134, + /* 560 */ 135, 370, 129, 343, 205, 206, 133, 20, 18, 416, + /* 570 */ 361, 418, 414, 23, 421, 422, 423, 424, 425, 426, + /* 580 */ 427, 428, 429, 430, 64, 136, 137, 37, 38, 3, + /* 590 */ 370, 41, 8, 9, 383, 384, 12, 13, 14, 15, + /* 600 */ 16, 362, 382, 20, 384, 348, 14, 15, 16, 370, + /* 610 */ 60, 61, 62, 63, 423, 166, 167, 378, 166, 8, + /* 620 */ 9, 172, 173, 12, 13, 14, 15, 16, 44, 35, + /* 630 */ 373, 20, 186, 236, 188, 186, 416, 188, 418, 342, + /* 640 */ 343, 421, 422, 423, 424, 425, 426, 100, 428, 166, + /* 650 */ 100, 431, 401, 433, 434, 435, 111, 406, 361, 439, + /* 660 */ 440, 215, 216, 371, 215, 216, 72, 218, 219, 220, /* 670 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 99, - /* 690 */ 141, 142, 102, 20, 393, 22, 395, 333, 423, 43, - /* 700 */ 0, 45, 46, 67, 68, 69, 33, 343, 35, 37, - /* 710 */ 74, 75, 76, 164, 342, 111, 80, 447, 342, 343, - /* 720 */ 450, 85, 86, 87, 88, 342, 343, 91, 178, 179, - /* 730 */ 180, 4, 59, 183, 370, 465, 466, 361, 383, 384, - /* 740 */ 470, 471, 72, 131, 370, 72, 382, 135, 384, 342, - /* 750 */ 343, 377, 1, 2, 204, 342, 343, 207, 371, 385, - /* 760 */ 210, 211, 212, 213, 214, 393, 333, 395, 361, 370, - /* 770 */ 97, 100, 401, 100, 361, 103, 377, 105, 106, 415, - /* 780 */ 108, 110, 418, 333, 385, 421, 422, 423, 424, 425, - /* 790 */ 426, 370, 428, 343, 0, 342, 343, 433, 377, 435, - /* 800 */ 348, 129, 252, 439, 440, 133, 385, 195, 196, 136, - /* 810 */ 137, 199, 350, 201, 361, 382, 452, 401, 447, 334, - /* 820 */ 370, 450, 355, 356, 460, 373, 443, 444, 445, 367, - /* 830 */ 447, 448, 382, 450, 384, 333, 465, 466, 376, 166, - /* 840 */ 167, 470, 471, 342, 343, 172, 173, 130, 465, 466, - /* 850 */ 342, 343, 101, 470, 471, 155, 14, 420, 236, 186, - /* 860 */ 238, 188, 20, 447, 164, 415, 450, 333, 418, 361, - /* 870 */ 371, 421, 422, 423, 424, 425, 426, 432, 428, 434, - /* 880 */ 44, 465, 466, 446, 382, 435, 470, 471, 215, 216, - /* 890 */ 440, 218, 219, 220, 221, 222, 223, 224, 225, 226, + /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 370, + /* 690 */ 168, 342, 22, 20, 350, 22, 377, 333, 447, 431, + /* 700 */ 447, 450, 434, 450, 385, 35, 33, 343, 35, 129, + /* 710 */ 130, 131, 132, 133, 134, 135, 465, 466, 465, 466, + /* 720 */ 376, 470, 471, 470, 471, 59, 355, 356, 178, 179, + /* 730 */ 180, 72, 59, 183, 370, 358, 370, 20, 432, 22, + /* 740 */ 434, 358, 393, 377, 395, 72, 382, 379, 384, 372, + /* 750 */ 382, 385, 35, 343, 204, 372, 370, 207, 420, 382, + /* 760 */ 210, 211, 212, 213, 214, 382, 100, 97, 102, 52, + /* 770 */ 97, 385, 342, 100, 420, 362, 130, 131, 333, 168, + /* 780 */ 416, 135, 418, 370, 446, 421, 422, 423, 424, 425, + /* 790 */ 426, 378, 428, 0, 35, 418, 419, 433, 388, 435, + /* 800 */ 446, 418, 252, 439, 440, 428, 333, 2, 333, 136, + /* 810 */ 137, 428, 39, 8, 9, 333, 452, 12, 13, 14, + /* 820 */ 15, 16, 2, 393, 460, 395, 129, 382, 8, 9, + /* 830 */ 133, 72, 12, 13, 14, 15, 16, 342, 343, 166, + /* 840 */ 167, 342, 343, 8, 9, 172, 173, 12, 13, 14, + /* 850 */ 15, 16, 97, 355, 356, 382, 361, 382, 272, 186, + /* 860 */ 361, 188, 250, 251, 382, 342, 343, 112, 113, 114, + /* 870 */ 115, 116, 117, 118, 119, 120, 121, 165, 123, 124, + /* 880 */ 125, 126, 127, 128, 361, 0, 342, 343, 215, 216, + /* 890 */ 333, 218, 219, 220, 221, 222, 223, 224, 225, 226, /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 12, - /* 910 */ 13, 432, 20, 434, 197, 198, 382, 20, 181, 22, - /* 920 */ 42, 333, 44, 129, 130, 131, 132, 133, 134, 135, - /* 930 */ 33, 343, 35, 8, 9, 342, 343, 12, 13, 14, - /* 940 */ 15, 16, 205, 206, 443, 444, 445, 333, 447, 448, - /* 950 */ 59, 371, 342, 343, 361, 401, 59, 343, 370, 345, - /* 960 */ 129, 130, 131, 132, 133, 134, 135, 362, 20, 72, - /* 970 */ 382, 361, 384, 342, 343, 370, 14, 15, 16, 420, - /* 980 */ 253, 8, 9, 378, 370, 12, 13, 14, 15, 16, - /* 990 */ 99, 129, 361, 102, 97, 133, 382, 100, 384, 342, - /* 1000 */ 343, 447, 333, 415, 450, 446, 418, 333, 166, 421, - /* 1010 */ 422, 423, 424, 425, 426, 0, 428, 343, 361, 465, - /* 1020 */ 466, 433, 39, 435, 470, 471, 333, 439, 440, 415, - /* 1030 */ 0, 350, 418, 136, 137, 421, 422, 423, 424, 425, - /* 1040 */ 426, 358, 428, 406, 370, 333, 21, 433, 460, 435, - /* 1050 */ 42, 382, 44, 439, 440, 372, 382, 376, 384, 34, - /* 1060 */ 168, 36, 343, 166, 167, 382, 2, 342, 343, 172, - /* 1070 */ 173, 165, 8, 9, 101, 382, 12, 13, 14, 15, - /* 1080 */ 16, 355, 356, 186, 371, 188, 361, 342, 343, 415, - /* 1090 */ 342, 343, 418, 168, 382, 421, 422, 423, 424, 425, - /* 1100 */ 426, 418, 428, 45, 46, 363, 361, 388, 366, 361, - /* 1110 */ 274, 428, 215, 216, 166, 218, 219, 220, 221, 222, + /* 910 */ 13, 130, 348, 45, 46, 100, 401, 20, 252, 22, + /* 920 */ 72, 333, 129, 130, 131, 132, 133, 134, 135, 365, + /* 930 */ 33, 343, 35, 342, 343, 342, 343, 373, 432, 382, + /* 940 */ 434, 342, 343, 14, 371, 342, 343, 333, 370, 20, + /* 950 */ 333, 239, 361, 4, 361, 377, 59, 343, 370, 345, + /* 960 */ 361, 249, 447, 385, 361, 450, 20, 371, 370, 72, + /* 970 */ 382, 333, 384, 333, 342, 343, 378, 333, 197, 198, + /* 980 */ 465, 466, 333, 44, 370, 470, 471, 443, 444, 445, + /* 990 */ 363, 447, 448, 366, 97, 371, 382, 100, 384, 382, + /* 1000 */ 185, 358, 187, 168, 416, 343, 418, 333, 333, 421, + /* 1010 */ 422, 423, 424, 425, 426, 372, 428, 343, 20, 333, + /* 1020 */ 382, 433, 382, 435, 251, 382, 382, 439, 440, 214, + /* 1030 */ 416, 382, 418, 136, 137, 421, 422, 423, 424, 425, + /* 1040 */ 426, 131, 428, 362, 370, 135, 20, 433, 460, 435, + /* 1050 */ 388, 370, 333, 439, 440, 333, 382, 382, 384, 378, + /* 1060 */ 13, 418, 419, 166, 167, 342, 343, 252, 382, 172, + /* 1070 */ 173, 428, 342, 343, 44, 443, 444, 445, 379, 447, + /* 1080 */ 448, 382, 35, 186, 361, 188, 136, 137, 342, 343, + /* 1090 */ 416, 361, 418, 333, 209, 421, 422, 423, 424, 425, + /* 1100 */ 426, 382, 428, 371, 382, 195, 196, 361, 343, 199, + /* 1110 */ 371, 201, 215, 216, 168, 218, 219, 220, 221, 222, /* 1120 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, - /* 1130 */ 233, 234, 235, 12, 13, 333, 462, 333, 342, 343, - /* 1140 */ 333, 20, 333, 22, 358, 239, 333, 343, 333, 345, - /* 1150 */ 342, 343, 342, 343, 33, 249, 35, 361, 372, 129, - /* 1160 */ 130, 131, 132, 133, 134, 135, 333, 431, 382, 361, - /* 1170 */ 434, 361, 250, 251, 370, 370, 379, 14, 379, 382, - /* 1180 */ 59, 382, 333, 20, 382, 358, 382, 333, 384, 382, - /* 1190 */ 385, 382, 358, 72, 0, 382, 44, 382, 333, 372, - /* 1200 */ 333, 333, 0, 371, 418, 419, 372, 357, 104, 382, - /* 1210 */ 360, 107, 111, 48, 428, 382, 382, 104, 97, 415, - /* 1220 */ 107, 100, 418, 333, 209, 421, 422, 423, 424, 425, - /* 1230 */ 426, 382, 428, 343, 251, 345, 382, 433, 44, 435, - /* 1240 */ 44, 22, 13, 439, 440, 418, 419, 382, 0, 382, - /* 1250 */ 382, 49, 418, 104, 35, 428, 107, 136, 137, 0, - /* 1260 */ 370, 160, 428, 104, 35, 44, 107, 0, 13, 200, - /* 1270 */ 22, 202, 382, 65, 384, 44, 59, 44, 44, 136, - /* 1280 */ 137, 22, 1, 2, 371, 47, 359, 166, 167, 22, - /* 1290 */ 35, 392, 474, 172, 173, 346, 457, 101, 463, 358, - /* 1300 */ 35, 35, 13, 358, 44, 415, 44, 186, 418, 188, - /* 1310 */ 370, 421, 422, 423, 424, 425, 426, 44, 428, 102, - /* 1320 */ 44, 44, 101, 433, 35, 435, 346, 44, 44, 439, - /* 1330 */ 440, 392, 101, 168, 101, 101, 215, 216, 100, 218, + /* 1130 */ 233, 234, 235, 12, 13, 461, 462, 333, 358, 342, + /* 1140 */ 343, 20, 382, 22, 342, 343, 0, 343, 111, 345, + /* 1150 */ 342, 343, 372, 388, 33, 333, 35, 44, 361, 342, + /* 1160 */ 343, 4, 382, 361, 379, 0, 168, 382, 357, 361, + /* 1170 */ 42, 360, 44, 42, 370, 44, 19, 200, 361, 202, + /* 1180 */ 59, 104, 104, 13, 107, 107, 382, 359, 384, 44, + /* 1190 */ 33, 0, 166, 72, 44, 49, 44, 160, 418, 401, + /* 1200 */ 104, 401, 253, 107, 382, 35, 49, 104, 428, 44, + /* 1210 */ 107, 54, 0, 22, 101, 276, 59, 35, 97, 0, + /* 1220 */ 416, 100, 418, 333, 44, 421, 422, 423, 424, 425, + /* 1230 */ 426, 401, 428, 343, 22, 345, 59, 433, 48, 435, + /* 1240 */ 44, 22, 44, 439, 440, 447, 101, 447, 450, 35, + /* 1250 */ 450, 101, 0, 101, 407, 334, 99, 136, 137, 102, + /* 1260 */ 370, 1, 2, 465, 466, 465, 466, 44, 470, 471, + /* 1270 */ 470, 471, 382, 100, 384, 44, 44, 447, 44, 102, + /* 1280 */ 450, 101, 463, 110, 392, 358, 474, 166, 167, 346, + /* 1290 */ 457, 358, 370, 172, 173, 465, 466, 101, 346, 101, + /* 1300 */ 470, 471, 50, 44, 274, 44, 416, 186, 418, 188, + /* 1310 */ 392, 421, 422, 423, 424, 425, 426, 44, 428, 44, + /* 1320 */ 44, 44, 44, 433, 101, 435, 341, 343, 381, 439, + /* 1330 */ 440, 449, 101, 101, 392, 101, 215, 216, 467, 218, /* 1340 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 333, 12, 13, - /* 1360 */ 35, 101, 44, 101, 44, 44, 0, 343, 22, 345, - /* 1370 */ 44, 343, 44, 341, 101, 381, 392, 101, 101, 33, - /* 1380 */ 44, 35, 449, 333, 101, 101, 467, 44, 441, 451, - /* 1390 */ 254, 417, 416, 343, 370, 345, 49, 72, 184, 403, - /* 1400 */ 42, 389, 20, 392, 165, 59, 382, 389, 384, 387, - /* 1410 */ 20, 98, 342, 342, 387, 389, 50, 354, 72, 101, - /* 1420 */ 370, 101, 101, 215, 387, 96, 353, 101, 276, 101, - /* 1430 */ 333, 342, 382, 95, 384, 352, 342, 101, 342, 415, - /* 1440 */ 343, 342, 418, 97, 101, 421, 422, 423, 424, 425, - /* 1450 */ 426, 333, 428, 188, 188, 20, 48, 433, 335, 435, - /* 1460 */ 335, 343, 20, 439, 440, 415, 339, 370, 418, 339, - /* 1470 */ 410, 421, 422, 423, 424, 425, 426, 20, 428, 382, - /* 1480 */ 20, 384, 350, 433, 384, 435, 344, 350, 370, 439, - /* 1490 */ 440, 20, 402, 53, 350, 344, 350, 350, 350, 342, - /* 1500 */ 382, 347, 384, 350, 335, 347, 342, 335, 370, 370, - /* 1510 */ 370, 370, 415, 382, 203, 418, 382, 382, 421, 422, - /* 1520 */ 423, 424, 425, 426, 414, 428, 370, 100, 412, 410, - /* 1530 */ 370, 370, 186, 415, 188, 348, 418, 370, 370, 421, - /* 1540 */ 422, 423, 424, 425, 426, 370, 428, 370, 192, 348, - /* 1550 */ 191, 433, 190, 435, 409, 333, 382, 439, 440, 261, - /* 1560 */ 384, 215, 216, 408, 456, 343, 342, 260, 397, 407, - /* 1570 */ 473, 392, 392, 382, 228, 229, 230, 231, 232, 233, - /* 1580 */ 234, 397, 456, 269, 177, 382, 382, 459, 271, 270, - /* 1590 */ 255, 458, 370, 278, 343, 417, 251, 475, 275, 20, - /* 1600 */ 273, 420, 344, 342, 382, 455, 384, 20, 395, 397, - /* 1610 */ 382, 348, 348, 397, 382, 170, 348, 382, 382, 382, - /* 1620 */ 394, 333, 456, 382, 454, 343, 453, 348, 100, 100, - /* 1630 */ 438, 343, 382, 342, 360, 366, 36, 415, 348, 336, - /* 1640 */ 418, 335, 398, 421, 422, 423, 424, 425, 426, 333, - /* 1650 */ 428, 469, 374, 468, 411, 433, 404, 435, 370, 343, - /* 1660 */ 364, 439, 440, 364, 364, 331, 0, 0, 398, 0, - /* 1670 */ 382, 349, 384, 42, 0, 35, 208, 35, 35, 35, - /* 1680 */ 208, 333, 0, 35, 35, 208, 370, 0, 208, 0, - /* 1690 */ 35, 343, 22, 0, 35, 195, 188, 186, 382, 0, - /* 1700 */ 384, 0, 0, 415, 0, 182, 418, 181, 0, 421, - /* 1710 */ 422, 423, 424, 425, 426, 0, 428, 0, 370, 47, - /* 1720 */ 0, 433, 0, 435, 0, 42, 0, 439, 440, 0, - /* 1730 */ 382, 415, 384, 0, 418, 0, 0, 421, 422, 423, - /* 1740 */ 424, 425, 426, 0, 428, 0, 155, 35, 0, 155, - /* 1750 */ 0, 435, 333, 0, 0, 439, 440, 139, 42, 0, - /* 1760 */ 0, 0, 343, 415, 0, 0, 418, 0, 0, 421, - /* 1770 */ 422, 423, 424, 425, 426, 0, 428, 0, 0, 0, - /* 1780 */ 333, 0, 0, 435, 0, 0, 0, 439, 440, 370, - /* 1790 */ 343, 0, 0, 0, 22, 0, 0, 0, 0, 333, - /* 1800 */ 0, 382, 22, 384, 48, 22, 48, 0, 59, 343, - /* 1810 */ 0, 35, 0, 0, 59, 333, 0, 370, 44, 14, - /* 1820 */ 47, 59, 0, 0, 0, 343, 39, 0, 39, 382, - /* 1830 */ 42, 384, 0, 39, 415, 177, 370, 418, 0, 0, - /* 1840 */ 421, 422, 423, 424, 425, 426, 47, 428, 382, 47, - /* 1850 */ 384, 40, 370, 0, 435, 0, 66, 375, 439, 440, - /* 1860 */ 35, 0, 415, 49, 382, 418, 384, 39, 421, 422, - /* 1870 */ 423, 424, 425, 426, 39, 428, 333, 35, 49, 0, - /* 1880 */ 35, 415, 49, 39, 418, 0, 343, 421, 422, 423, - /* 1890 */ 424, 425, 426, 35, 428, 49, 430, 415, 1, 333, - /* 1900 */ 418, 39, 0, 421, 422, 423, 424, 425, 426, 343, - /* 1910 */ 428, 464, 0, 370, 0, 0, 19, 109, 375, 35, - /* 1920 */ 22, 0, 107, 35, 35, 382, 35, 384, 35, 35, - /* 1930 */ 33, 35, 22, 0, 35, 22, 370, 51, 35, 44, - /* 1940 */ 35, 375, 35, 0, 44, 22, 49, 0, 382, 22, - /* 1950 */ 384, 35, 55, 56, 57, 58, 59, 35, 415, 0, - /* 1960 */ 0, 418, 0, 35, 421, 422, 423, 424, 425, 426, - /* 1970 */ 22, 428, 20, 35, 35, 35, 101, 100, 0, 35, - /* 1980 */ 100, 415, 0, 22, 418, 168, 22, 421, 422, 423, - /* 1990 */ 424, 425, 426, 333, 428, 189, 99, 0, 168, 102, - /* 2000 */ 0, 170, 193, 343, 168, 3, 100, 44, 256, 175, - /* 2010 */ 101, 100, 44, 48, 48, 98, 96, 44, 44, 101, - /* 2020 */ 101, 44, 47, 47, 100, 333, 100, 256, 3, 100, - /* 2030 */ 370, 134, 101, 101, 100, 343, 101, 44, 35, 47, - /* 2040 */ 35, 35, 382, 333, 384, 35, 35, 35, 47, 0, - /* 2050 */ 101, 101, 44, 343, 0, 0, 0, 256, 100, 39, - /* 2060 */ 47, 171, 370, 0, 110, 39, 169, 100, 250, 47, - /* 2070 */ 101, 174, 101, 169, 382, 415, 384, 44, 418, 100, - /* 2080 */ 370, 421, 422, 423, 424, 425, 426, 100, 428, 100, - /* 2090 */ 333, 194, 382, 237, 384, 100, 98, 98, 2, 22, - /* 2100 */ 343, 22, 215, 35, 47, 47, 35, 415, 101, 100, - /* 2110 */ 418, 35, 122, 421, 422, 423, 424, 425, 426, 100, - /* 2120 */ 428, 35, 101, 100, 100, 415, 111, 370, 418, 101, - /* 2130 */ 100, 421, 422, 423, 424, 425, 426, 333, 428, 382, - /* 2140 */ 101, 384, 217, 100, 35, 101, 100, 343, 101, 101, - /* 2150 */ 100, 100, 35, 101, 101, 44, 122, 122, 122, 333, - /* 2160 */ 100, 100, 100, 35, 100, 100, 22, 66, 65, 343, - /* 2170 */ 35, 35, 415, 44, 370, 418, 35, 35, 421, 422, - /* 2180 */ 423, 424, 425, 426, 333, 428, 382, 35, 384, 35, - /* 2190 */ 35, 35, 72, 94, 343, 35, 370, 35, 35, 22, - /* 2200 */ 35, 35, 35, 72, 35, 35, 333, 35, 382, 35, - /* 2210 */ 384, 35, 22, 35, 0, 35, 343, 49, 39, 415, - /* 2220 */ 0, 370, 418, 35, 49, 421, 422, 423, 424, 425, - /* 2230 */ 426, 39, 428, 382, 0, 384, 35, 39, 49, 0, - /* 2240 */ 35, 415, 49, 370, 418, 39, 0, 421, 422, 423, - /* 2250 */ 424, 425, 426, 35, 428, 382, 35, 384, 22, 21, - /* 2260 */ 476, 476, 22, 22, 21, 20, 415, 0, 476, 418, - /* 2270 */ 333, 476, 421, 422, 423, 424, 425, 426, 476, 428, - /* 2280 */ 343, 476, 476, 476, 476, 476, 476, 476, 415, 476, - /* 2290 */ 333, 418, 476, 476, 421, 422, 423, 424, 425, 426, - /* 2300 */ 343, 428, 476, 476, 476, 333, 476, 370, 476, 476, - /* 2310 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 382, - /* 2320 */ 476, 384, 476, 476, 476, 476, 476, 370, 476, 476, - /* 2330 */ 476, 476, 476, 476, 476, 476, 476, 333, 476, 382, - /* 2340 */ 476, 384, 370, 476, 476, 476, 476, 343, 476, 476, - /* 2350 */ 476, 476, 415, 476, 382, 418, 384, 476, 421, 422, - /* 2360 */ 423, 424, 425, 426, 476, 428, 476, 476, 476, 476, - /* 2370 */ 476, 476, 415, 476, 370, 418, 476, 476, 421, 422, - /* 2380 */ 423, 424, 425, 426, 476, 428, 382, 415, 384, 476, - /* 2390 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, - /* 2400 */ 428, 476, 476, 476, 476, 333, 476, 476, 476, 476, - /* 2410 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 415, - /* 2420 */ 476, 476, 418, 476, 333, 421, 422, 423, 424, 425, - /* 2430 */ 426, 476, 428, 476, 343, 476, 476, 476, 476, 476, - /* 2440 */ 476, 476, 370, 476, 476, 476, 476, 476, 476, 476, - /* 2450 */ 476, 476, 333, 476, 382, 476, 384, 476, 476, 476, - /* 2460 */ 476, 370, 343, 476, 476, 476, 476, 476, 476, 476, - /* 2470 */ 476, 333, 476, 382, 476, 384, 476, 476, 476, 476, - /* 2480 */ 476, 343, 476, 476, 476, 476, 476, 415, 476, 370, - /* 2490 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, - /* 2500 */ 428, 382, 476, 384, 476, 476, 415, 476, 370, 418, - /* 2510 */ 476, 476, 421, 422, 423, 424, 425, 426, 333, 428, - /* 2520 */ 382, 476, 384, 476, 476, 476, 476, 476, 343, 476, - /* 2530 */ 476, 476, 476, 476, 415, 476, 476, 418, 476, 476, - /* 2540 */ 421, 422, 423, 424, 425, 426, 476, 428, 476, 476, - /* 2550 */ 476, 476, 476, 415, 476, 370, 418, 476, 476, 421, - /* 2560 */ 422, 423, 424, 425, 426, 476, 428, 382, 476, 384, - /* 2570 */ 476, 476, 476, 476, 476, 333, 476, 476, 476, 476, - /* 2580 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 476, - /* 2590 */ 476, 333, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2600 */ 415, 343, 476, 418, 476, 476, 421, 422, 423, 424, - /* 2610 */ 425, 426, 370, 428, 476, 476, 476, 476, 476, 476, - /* 2620 */ 476, 476, 476, 476, 382, 476, 384, 476, 370, 476, - /* 2630 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2640 */ 382, 333, 384, 476, 476, 476, 476, 476, 476, 476, - /* 2650 */ 476, 343, 476, 476, 476, 476, 476, 415, 476, 476, - /* 2660 */ 418, 333, 476, 421, 422, 423, 424, 425, 426, 476, - /* 2670 */ 428, 343, 476, 415, 476, 476, 418, 476, 370, 421, - /* 2680 */ 422, 423, 424, 425, 426, 476, 428, 476, 476, 476, - /* 2690 */ 382, 476, 384, 476, 476, 476, 476, 476, 370, 476, - /* 2700 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, - /* 2710 */ 382, 476, 384, 476, 476, 476, 476, 476, 343, 476, - /* 2720 */ 476, 476, 476, 415, 476, 476, 418, 476, 476, 421, - /* 2730 */ 422, 423, 424, 425, 426, 476, 428, 476, 476, 476, - /* 2740 */ 476, 476, 33, 415, 476, 370, 418, 476, 476, 421, - /* 2750 */ 422, 423, 424, 425, 426, 476, 428, 382, 49, 384, - /* 2760 */ 476, 476, 476, 476, 55, 56, 57, 58, 59, 476, - /* 2770 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2780 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2790 */ 415, 476, 476, 418, 476, 476, 421, 422, 423, 424, - /* 2800 */ 425, 426, 476, 428, 476, 476, 476, 476, 99, 476, - /* 2810 */ 476, 102, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2820 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2830 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2840 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2850 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 333, 168, 44, + /* 1360 */ 101, 441, 101, 13, 44, 451, 13, 343, 254, 345, + /* 1370 */ 188, 402, 49, 417, 101, 404, 101, 101, 101, 101, + /* 1380 */ 42, 184, 389, 333, 20, 35, 392, 165, 35, 389, + /* 1390 */ 387, 20, 387, 343, 370, 345, 342, 342, 389, 387, + /* 1400 */ 98, 96, 188, 354, 353, 342, 382, 95, 384, 352, + /* 1410 */ 342, 342, 342, 20, 335, 48, 101, 335, 20, 339, + /* 1420 */ 370, 101, 339, 350, 411, 20, 384, 20, 344, 20, + /* 1430 */ 403, 350, 382, 344, 384, 350, 350, 350, 350, 342, + /* 1440 */ 416, 350, 418, 53, 335, 421, 422, 423, 424, 425, + /* 1450 */ 426, 347, 428, 347, 382, 333, 382, 433, 382, 435, + /* 1460 */ 370, 342, 370, 439, 440, 343, 416, 345, 418, 370, + /* 1470 */ 370, 421, 422, 423, 424, 425, 426, 370, 428, 335, + /* 1480 */ 370, 333, 370, 433, 203, 435, 370, 370, 370, 439, + /* 1490 */ 440, 343, 370, 370, 100, 348, 192, 191, 190, 384, + /* 1500 */ 342, 348, 261, 392, 382, 410, 384, 413, 415, 411, + /* 1510 */ 409, 382, 456, 408, 333, 260, 392, 382, 370, 382, + /* 1520 */ 456, 382, 402, 397, 343, 269, 459, 397, 177, 458, + /* 1530 */ 382, 456, 384, 271, 270, 255, 402, 475, 416, 273, + /* 1540 */ 418, 278, 275, 421, 422, 423, 424, 425, 426, 251, + /* 1550 */ 428, 370, 343, 20, 344, 433, 454, 435, 420, 342, + /* 1560 */ 20, 439, 440, 382, 416, 384, 418, 395, 402, 421, + /* 1570 */ 422, 423, 424, 425, 426, 348, 428, 469, 348, 366, + /* 1580 */ 455, 433, 397, 435, 382, 453, 382, 439, 440, 382, + /* 1590 */ 382, 333, 397, 382, 170, 468, 348, 416, 382, 418, + /* 1600 */ 394, 343, 421, 422, 423, 424, 425, 426, 100, 428, + /* 1610 */ 100, 343, 438, 348, 433, 382, 435, 374, 348, 36, + /* 1620 */ 439, 440, 333, 336, 342, 335, 405, 412, 370, 360, + /* 1630 */ 331, 0, 343, 398, 398, 364, 364, 349, 364, 0, + /* 1640 */ 382, 0, 384, 42, 0, 35, 208, 35, 35, 35, + /* 1650 */ 333, 208, 0, 35, 35, 208, 0, 208, 0, 370, + /* 1660 */ 343, 35, 0, 0, 22, 35, 195, 188, 186, 0, + /* 1670 */ 0, 382, 0, 384, 416, 182, 418, 181, 0, 421, + /* 1680 */ 422, 423, 424, 425, 426, 333, 428, 370, 0, 0, + /* 1690 */ 47, 433, 0, 435, 0, 343, 42, 439, 440, 382, + /* 1700 */ 0, 384, 0, 0, 0, 416, 0, 418, 0, 0, + /* 1710 */ 421, 422, 423, 424, 425, 426, 0, 428, 35, 155, + /* 1720 */ 0, 155, 370, 0, 435, 0, 0, 0, 439, 440, + /* 1730 */ 0, 0, 0, 416, 382, 418, 384, 0, 421, 422, + /* 1740 */ 423, 424, 425, 426, 0, 428, 0, 0, 0, 0, + /* 1750 */ 0, 0, 435, 0, 42, 0, 439, 440, 0, 0, + /* 1760 */ 0, 0, 0, 22, 0, 0, 0, 139, 416, 48, + /* 1770 */ 418, 0, 0, 421, 422, 423, 424, 425, 426, 333, + /* 1780 */ 428, 48, 0, 59, 0, 22, 0, 435, 59, 343, + /* 1790 */ 22, 439, 440, 0, 59, 0, 42, 14, 0, 333, + /* 1800 */ 44, 0, 35, 0, 47, 0, 177, 0, 0, 343, + /* 1810 */ 333, 0, 39, 0, 0, 40, 370, 47, 39, 39, + /* 1820 */ 343, 47, 35, 0, 39, 35, 49, 49, 382, 0, + /* 1830 */ 384, 39, 35, 39, 0, 35, 370, 49, 49, 0, + /* 1840 */ 0, 39, 66, 0, 0, 109, 107, 370, 382, 22, + /* 1850 */ 384, 35, 0, 35, 35, 35, 35, 35, 22, 382, + /* 1860 */ 35, 384, 416, 35, 418, 35, 0, 421, 422, 423, + /* 1870 */ 424, 425, 426, 35, 428, 22, 44, 44, 0, 22, + /* 1880 */ 35, 435, 416, 0, 418, 22, 440, 421, 422, 423, + /* 1890 */ 424, 425, 426, 416, 428, 418, 333, 51, 421, 422, + /* 1900 */ 423, 424, 425, 426, 35, 428, 343, 333, 0, 35, + /* 1910 */ 0, 35, 0, 22, 20, 35, 35, 343, 101, 35, + /* 1920 */ 0, 193, 333, 22, 100, 35, 100, 0, 462, 189, + /* 1930 */ 0, 22, 343, 370, 0, 3, 44, 256, 101, 98, + /* 1940 */ 44, 464, 100, 48, 370, 382, 48, 384, 100, 375, + /* 1950 */ 96, 170, 101, 47, 47, 44, 382, 101, 384, 370, + /* 1960 */ 100, 100, 100, 44, 168, 101, 175, 168, 100, 44, + /* 1970 */ 256, 382, 168, 384, 101, 3, 101, 44, 256, 416, + /* 1980 */ 35, 418, 35, 35, 421, 422, 423, 424, 425, 426, + /* 1990 */ 416, 428, 418, 333, 35, 421, 422, 423, 424, 425, + /* 2000 */ 426, 35, 428, 343, 35, 416, 101, 418, 333, 47, + /* 2010 */ 421, 422, 423, 424, 425, 426, 1, 428, 343, 430, + /* 2020 */ 44, 47, 101, 0, 0, 0, 0, 39, 100, 47, + /* 2030 */ 370, 100, 0, 39, 19, 375, 473, 101, 101, 100, + /* 2040 */ 100, 171, 382, 100, 384, 370, 169, 47, 33, 100, + /* 2050 */ 375, 110, 44, 98, 98, 2, 250, 382, 22, 384, + /* 2060 */ 100, 100, 215, 100, 49, 101, 101, 333, 101, 47, + /* 2070 */ 55, 56, 57, 58, 59, 100, 416, 343, 418, 100, + /* 2080 */ 47, 421, 422, 423, 424, 425, 426, 237, 428, 22, + /* 2090 */ 101, 416, 100, 418, 111, 35, 421, 422, 423, 424, + /* 2100 */ 425, 426, 333, 428, 370, 35, 217, 101, 101, 100, + /* 2110 */ 35, 100, 343, 101, 99, 35, 382, 102, 384, 100, + /* 2120 */ 35, 35, 101, 122, 122, 122, 44, 100, 35, 101, + /* 2130 */ 100, 100, 122, 22, 100, 66, 65, 100, 35, 370, + /* 2140 */ 35, 35, 35, 35, 35, 35, 72, 35, 94, 134, + /* 2150 */ 416, 382, 418, 384, 22, 421, 422, 423, 424, 425, + /* 2160 */ 426, 44, 428, 35, 333, 35, 35, 72, 35, 35, + /* 2170 */ 35, 35, 35, 35, 343, 35, 35, 22, 35, 0, + /* 2180 */ 35, 39, 0, 35, 169, 416, 0, 418, 49, 174, + /* 2190 */ 421, 422, 423, 424, 425, 426, 49, 428, 39, 35, + /* 2200 */ 49, 370, 39, 0, 35, 49, 39, 0, 35, 194, + /* 2210 */ 35, 0, 22, 382, 21, 384, 22, 22, 21, 20, + /* 2220 */ 476, 333, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2230 */ 476, 343, 333, 476, 476, 476, 476, 476, 476, 476, + /* 2240 */ 476, 476, 343, 476, 476, 476, 476, 416, 476, 418, + /* 2250 */ 476, 333, 421, 422, 423, 424, 425, 426, 370, 428, + /* 2260 */ 476, 343, 476, 476, 476, 476, 476, 476, 476, 370, + /* 2270 */ 382, 476, 384, 476, 476, 476, 476, 476, 476, 476, + /* 2280 */ 476, 382, 476, 384, 476, 476, 476, 476, 370, 476, + /* 2290 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2300 */ 382, 476, 384, 476, 416, 476, 418, 476, 476, 421, + /* 2310 */ 422, 423, 424, 425, 426, 416, 428, 418, 333, 476, + /* 2320 */ 421, 422, 423, 424, 425, 426, 476, 428, 343, 476, + /* 2330 */ 476, 476, 476, 476, 416, 476, 418, 333, 476, 421, + /* 2340 */ 422, 423, 424, 425, 426, 476, 428, 343, 476, 476, + /* 2350 */ 476, 476, 476, 333, 476, 370, 476, 476, 476, 476, + /* 2360 */ 476, 476, 476, 343, 476, 476, 476, 382, 333, 384, + /* 2370 */ 476, 476, 476, 476, 370, 476, 476, 476, 343, 476, + /* 2380 */ 476, 476, 476, 476, 333, 476, 382, 476, 384, 476, + /* 2390 */ 370, 476, 476, 476, 343, 476, 476, 476, 476, 476, + /* 2400 */ 476, 416, 382, 418, 384, 370, 421, 422, 423, 424, + /* 2410 */ 425, 426, 476, 428, 476, 476, 476, 382, 476, 384, + /* 2420 */ 416, 370, 418, 476, 476, 421, 422, 423, 424, 425, + /* 2430 */ 426, 476, 428, 382, 476, 384, 416, 476, 418, 476, + /* 2440 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2450 */ 476, 416, 333, 418, 476, 476, 421, 422, 423, 424, + /* 2460 */ 425, 426, 343, 428, 476, 476, 476, 416, 476, 418, + /* 2470 */ 476, 476, 421, 422, 423, 424, 425, 426, 476, 428, + /* 2480 */ 476, 476, 476, 333, 476, 476, 476, 476, 476, 370, + /* 2490 */ 476, 476, 476, 343, 476, 476, 476, 476, 476, 476, + /* 2500 */ 476, 382, 476, 384, 476, 476, 476, 476, 476, 476, + /* 2510 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2520 */ 370, 476, 476, 476, 476, 476, 476, 476, 343, 476, + /* 2530 */ 476, 476, 382, 476, 384, 416, 476, 418, 476, 476, + /* 2540 */ 421, 422, 423, 424, 425, 426, 476, 428, 476, 333, + /* 2550 */ 476, 476, 476, 476, 476, 370, 476, 476, 476, 343, + /* 2560 */ 476, 476, 476, 476, 476, 476, 416, 382, 418, 384, + /* 2570 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2580 */ 476, 476, 476, 476, 476, 476, 370, 476, 476, 476, + /* 2590 */ 476, 476, 476, 476, 476, 476, 476, 476, 382, 476, + /* 2600 */ 384, 416, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2610 */ 425, 426, 476, 428, 476, 333, 476, 476, 476, 476, + /* 2620 */ 476, 476, 476, 476, 476, 343, 476, 476, 476, 476, + /* 2630 */ 476, 476, 416, 476, 418, 476, 333, 421, 422, 423, + /* 2640 */ 424, 425, 426, 476, 428, 476, 343, 476, 476, 476, + /* 2650 */ 476, 476, 370, 476, 476, 476, 476, 476, 476, 476, + /* 2660 */ 476, 476, 333, 476, 382, 476, 384, 476, 476, 476, + /* 2670 */ 476, 476, 343, 370, 476, 476, 476, 476, 476, 476, + /* 2680 */ 476, 333, 476, 476, 476, 382, 476, 384, 476, 476, + /* 2690 */ 476, 343, 476, 476, 476, 476, 476, 476, 416, 370, + /* 2700 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2710 */ 428, 382, 333, 384, 476, 476, 476, 476, 370, 416, + /* 2720 */ 476, 418, 343, 476, 421, 422, 423, 424, 425, 426, + /* 2730 */ 382, 428, 384, 476, 476, 476, 476, 476, 333, 476, + /* 2740 */ 476, 476, 476, 476, 476, 416, 476, 418, 343, 370, + /* 2750 */ 421, 422, 423, 424, 425, 426, 476, 428, 476, 476, + /* 2760 */ 476, 382, 476, 384, 416, 476, 418, 476, 476, 421, + /* 2770 */ 422, 423, 424, 425, 426, 370, 428, 476, 476, 476, + /* 2780 */ 476, 476, 476, 476, 476, 476, 476, 382, 476, 384, + /* 2790 */ 476, 476, 476, 476, 476, 416, 476, 418, 476, 476, + /* 2800 */ 421, 422, 423, 424, 425, 426, 333, 428, 476, 476, + /* 2810 */ 476, 476, 476, 476, 476, 476, 343, 476, 476, 476, + /* 2820 */ 476, 416, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2830 */ 425, 426, 476, 428, 476, 476, 476, 476, 476, 476, + /* 2840 */ 476, 476, 476, 370, 476, 476, 476, 476, 476, 476, + /* 2850 */ 476, 476, 476, 476, 476, 382, 476, 384, 476, 476, /* 2860 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2870 */ 476, 476, 476, 476, 476, 476, 476, 476, 169, 170, - /* 2880 */ 476, 476, 476, 174, 476, 176, 476, 476, 476, 476, - /* 2890 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2900 */ 476, 476, 476, 194, 476, 476, 476, 476, 476, 476, - /* 2910 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2920 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2930 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2940 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2950 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2960 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2970 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2980 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2990 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3000 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3010 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3020 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3030 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3040 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3050 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3060 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 3070 */ 476, 476, 476, 330, 330, 330, 330, 330, 330, 330, + /* 2870 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2880 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 416, + /* 2890 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2900 */ 476, 428, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2910 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2920 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2930 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2940 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2950 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2960 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2970 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2980 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 2990 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3000 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3010 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3020 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3030 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3040 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3050 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3060 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, + /* 3070 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3080 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3090 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3100 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, @@ -836,205 +836,205 @@ static const YYCODETYPE yy_lookahead[] = { /* 3200 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3210 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, /* 3220 */ 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, - /* 3230 */ 330, 330, 330, 330, + /* 3230 */ 330, 330, }; -#define YY_SHIFT_COUNT (761) +#define YY_SHIFT_COUNT (763) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2709) +#define YY_SHIFT_MAX (2211) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 550, 0, 224, 0, 449, 449, 449, 449, 449, 449, /* 10 */ 449, 449, 449, 449, 449, 449, 673, 897, 897, 1121, /* 20 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, /* 30 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 148, - /* 50 */ 368, 93, 64, 140, 147, 322, 147, 64, 64, 1346, - /* 60 */ 1346, 1346, 147, 1346, 1346, 314, 147, 76, 369, 111, - /* 70 */ 111, 369, 656, 656, 18, 327, 7, 7, 111, 111, - /* 80 */ 111, 111, 111, 111, 111, 145, 111, 111, 248, 76, - /* 90 */ 111, 111, 363, 111, 76, 111, 145, 111, 145, 76, - /* 100 */ 111, 111, 76, 111, 76, 76, 76, 111, 343, 223, - /* 110 */ 189, 189, 405, 114, 443, 443, 443, 443, 443, 443, - /* 120 */ 443, 443, 443, 443, 443, 443, 443, 443, 443, 443, - /* 130 */ 443, 443, 443, 672, 483, 18, 327, 293, 253, 253, - /* 140 */ 253, 217, 622, 622, 293, 523, 523, 523, 248, 460, - /* 150 */ 354, 76, 585, 76, 585, 585, 604, 670, 35, 35, - /* 160 */ 35, 35, 35, 35, 35, 35, 1897, 636, 15, 167, - /* 170 */ 16, 297, 515, 252, 454, 454, 373, 842, 389, 484, - /* 180 */ 1058, 1163, 862, 892, 922, 983, 45, 922, 878, 727, - /* 190 */ 948, 1136, 1347, 1214, 1358, 1382, 1358, 1239, 1390, 1390, - /* 200 */ 1358, 1239, 1239, 1313, 1329, 1390, 1338, 1390, 1390, 1390, - /* 210 */ 1435, 1408, 1435, 1408, 1442, 248, 1457, 248, 1460, 1471, - /* 220 */ 248, 1460, 248, 248, 248, 1390, 248, 1440, 1440, 1435, - /* 230 */ 76, 76, 76, 76, 76, 76, 76, 76, 76, 76, - /* 240 */ 76, 1390, 1435, 585, 585, 585, 1311, 1427, 1442, 343, - /* 250 */ 1356, 1359, 1457, 343, 1362, 1390, 1382, 1382, 585, 1298, - /* 260 */ 1307, 585, 1298, 1307, 585, 585, 76, 1314, 1407, 1298, - /* 270 */ 1317, 1319, 1335, 1136, 1315, 1323, 1327, 1345, 523, 1579, - /* 280 */ 1390, 1460, 343, 343, 1587, 1307, 585, 585, 585, 585, - /* 290 */ 585, 1307, 585, 1445, 343, 604, 343, 523, 1528, 1529, - /* 300 */ 585, 670, 1390, 343, 1600, 1435, 2904, 2904, 2904, 2904, - /* 310 */ 2904, 2904, 2904, 2904, 2904, 34, 2709, 238, 590, 198, - /* 320 */ 539, 973, 794, 486, 1064, 925, 1030, 48, 48, 48, - /* 330 */ 48, 48, 48, 48, 48, 48, 831, 612, 65, 65, - /* 340 */ 549, 737, 700, 891, 262, 448, 1025, 717, 531, 531, - /* 350 */ 962, 751, 906, 962, 962, 962, 1202, 1015, 513, 1219, - /* 360 */ 1008, 1101, 1194, 1104, 1113, 1149, 1159, 381, 1229, 1248, - /* 370 */ 1259, 1267, 1069, 1196, 1221, 1217, 1231, 1233, 1234, 1143, - /* 380 */ 836, 1152, 1165, 1260, 1262, 1273, 1276, 1277, 1318, 1281, - /* 390 */ 1283, 1208, 1284, 1238, 1320, 1321, 1326, 1328, 1336, 1343, - /* 400 */ 671, 1265, 1266, 1255, 1289, 1325, 1366, 1666, 1667, 1669, - /* 410 */ 1631, 1674, 1640, 1468, 1642, 1643, 1644, 1472, 1682, 1648, - /* 420 */ 1649, 1477, 1687, 1480, 1689, 1655, 1699, 1670, 1693, 1659, - /* 430 */ 1500, 1508, 1511, 1701, 1702, 1704, 1523, 1526, 1708, 1715, - /* 440 */ 1672, 1717, 1720, 1722, 1683, 1724, 1726, 1729, 1733, 1735, - /* 450 */ 1736, 1743, 1745, 1591, 1712, 1748, 1594, 1750, 1753, 1754, - /* 460 */ 1764, 1765, 1767, 1768, 1775, 1777, 1778, 1779, 1781, 1782, - /* 470 */ 1784, 1785, 1786, 1716, 1759, 1760, 1761, 1791, 1792, 1793, - /* 480 */ 1772, 1795, 1796, 1797, 1618, 1798, 1800, 1780, 1756, 1783, - /* 490 */ 1758, 1807, 1749, 1776, 1810, 1755, 1812, 1762, 1813, 1816, - /* 500 */ 1788, 1787, 1774, 1773, 1799, 1805, 1802, 1822, 1811, 1789, - /* 510 */ 1823, 1824, 1827, 1794, 1658, 1832, 1838, 1839, 1790, 1853, - /* 520 */ 1855, 1825, 1814, 1828, 1861, 1842, 1829, 1835, 1879, 1845, - /* 530 */ 1833, 1844, 1885, 1858, 1846, 1862, 1902, 1912, 1914, 1915, - /* 540 */ 1808, 1815, 1884, 1898, 1921, 1888, 1889, 1891, 1893, 1894, - /* 550 */ 1896, 1899, 1895, 1900, 1903, 1905, 1910, 1907, 1933, 1913, - /* 560 */ 1943, 1923, 1886, 1947, 1927, 1916, 1959, 1922, 1960, 1928, - /* 570 */ 1962, 1948, 1952, 1938, 1939, 1940, 1875, 1877, 1978, 1817, - /* 580 */ 1880, 1809, 1944, 1961, 1982, 1806, 1964, 1830, 1831, 1997, - /* 590 */ 2000, 1836, 1834, 2002, 1963, 1752, 1906, 1909, 1911, 1965, - /* 600 */ 1917, 1966, 1920, 1918, 1968, 1973, 1919, 1924, 1926, 1929, - /* 610 */ 1931, 1974, 1975, 1976, 1934, 1977, 1771, 1932, 1935, 2025, - /* 620 */ 1993, 1801, 2003, 2005, 2006, 2010, 2011, 2012, 1949, 1950, - /* 630 */ 1992, 1818, 2008, 2001, 2049, 2054, 2055, 2056, 1958, 2020, - /* 640 */ 1773, 2013, 1967, 1969, 1971, 1979, 1987, 1890, 1989, 2063, - /* 650 */ 2026, 1904, 1995, 1954, 1773, 2022, 2033, 1998, 1856, 1999, - /* 660 */ 2096, 2077, 1887, 2009, 2007, 2019, 2021, 2023, 2028, 2057, - /* 670 */ 2024, 2030, 2058, 2039, 2079, 1925, 2043, 2015, 2044, 2068, - /* 680 */ 2071, 2046, 2047, 2076, 2050, 2048, 2086, 2051, 2052, 2109, - /* 690 */ 2060, 2053, 2117, 2061, 1990, 2034, 2035, 2036, 2062, 2111, - /* 700 */ 2064, 2128, 2065, 2111, 2111, 2144, 2101, 2103, 2135, 2136, - /* 710 */ 2141, 2142, 2152, 2154, 2155, 2156, 2120, 2099, 2129, 2160, - /* 720 */ 2162, 2163, 2177, 2165, 2166, 2167, 2131, 1895, 2169, 1900, - /* 730 */ 2170, 2172, 2174, 2176, 2190, 2178, 2214, 2180, 2168, 2179, - /* 740 */ 2220, 2188, 2175, 2192, 2234, 2201, 2189, 2198, 2239, 2205, - /* 750 */ 2193, 2206, 2246, 2218, 2221, 2267, 2236, 2238, 2240, 2241, - /* 760 */ 2243, 2245, + /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 113, + /* 50 */ 165, 815, 547, 666, 112, 215, 112, 547, 547, 117, + /* 60 */ 117, 117, 112, 117, 117, 6, 112, 128, 222, 162, + /* 70 */ 162, 222, 236, 236, 190, 271, 247, 247, 162, 162, + /* 80 */ 162, 162, 162, 162, 162, 296, 162, 162, 4, 128, + /* 90 */ 162, 162, 303, 162, 128, 162, 296, 162, 296, 128, + /* 100 */ 162, 162, 128, 162, 128, 128, 128, 162, 70, 223, + /* 110 */ 189, 189, 226, 130, 446, 446, 446, 446, 446, 446, + /* 120 */ 446, 446, 446, 446, 446, 446, 446, 446, 446, 446, + /* 130 */ 446, 446, 446, 433, 315, 190, 271, 594, 522, 522, + /* 140 */ 522, 520, 13, 13, 594, 583, 583, 583, 4, 98, + /* 150 */ 397, 128, 659, 128, 659, 659, 545, 848, 755, 755, + /* 160 */ 755, 755, 755, 755, 755, 755, 2015, 424, 501, 611, + /* 170 */ 15, 212, 717, 230, 307, 307, 452, 483, 277, 946, + /* 180 */ 868, 929, 697, 998, 612, 773, 586, 612, 1128, 949, + /* 190 */ 1026, 1114, 1323, 1197, 1338, 1364, 1338, 1222, 1371, 1371, + /* 200 */ 1338, 1222, 1222, 1302, 1305, 1371, 1312, 1371, 1371, 1371, + /* 210 */ 1393, 1367, 1393, 1367, 1398, 4, 1405, 4, 1407, 1409, + /* 220 */ 4, 1407, 4, 4, 4, 1371, 4, 1390, 1390, 1393, + /* 230 */ 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + /* 240 */ 128, 1371, 1393, 659, 659, 659, 1281, 1394, 1398, 70, + /* 250 */ 1304, 1306, 1405, 70, 1308, 1114, 1371, 1364, 1364, 659, + /* 260 */ 1241, 1255, 659, 1241, 1255, 659, 659, 128, 1256, 1351, + /* 270 */ 1241, 1262, 1264, 1280, 1114, 1263, 1267, 1266, 1298, 583, + /* 280 */ 1533, 1114, 1371, 1407, 70, 70, 1540, 1255, 659, 659, + /* 290 */ 659, 659, 659, 1255, 659, 1424, 70, 545, 70, 583, + /* 300 */ 1508, 1510, 659, 848, 1371, 70, 1583, 1393, 2902, 2902, + /* 310 */ 2902, 2902, 2902, 2902, 2902, 2902, 2902, 34, 229, 51, + /* 320 */ 1157, 31, 584, 406, 425, 805, 820, 835, 793, 48, + /* 330 */ 48, 48, 48, 48, 48, 48, 48, 48, 580, 910, + /* 340 */ 3, 3, 166, 359, 32, 29, 367, 670, 37, 781, + /* 350 */ 646, 646, 592, 308, 712, 592, 592, 592, 1146, 885, + /* 360 */ 1113, 238, 1131, 1037, 1165, 1077, 1078, 1096, 1103, 1047, + /* 370 */ 1170, 1191, 1212, 1219, 977, 1145, 1150, 1177, 1152, 1180, + /* 380 */ 1196, 950, 1030, 939, 1190, 1198, 1223, 1231, 1232, 1234, + /* 390 */ 1259, 1260, 1261, 287, 1273, 416, 1275, 1276, 1277, 1278, + /* 400 */ 1315, 1320, 1173, 1182, 1214, 1350, 1353, 759, 1252, 1631, + /* 410 */ 1639, 1641, 1601, 1644, 1610, 1438, 1612, 1613, 1614, 1443, + /* 420 */ 1652, 1618, 1619, 1447, 1656, 1449, 1658, 1626, 1662, 1642, + /* 430 */ 1663, 1630, 1471, 1479, 1482, 1669, 1670, 1672, 1493, 1496, + /* 440 */ 1678, 1688, 1643, 1689, 1692, 1694, 1654, 1700, 1702, 1703, + /* 450 */ 1704, 1706, 1708, 1709, 1716, 1564, 1683, 1720, 1566, 1723, + /* 460 */ 1725, 1726, 1727, 1730, 1731, 1732, 1737, 1744, 1746, 1747, + /* 470 */ 1748, 1749, 1750, 1751, 1753, 1712, 1755, 1758, 1759, 1760, + /* 480 */ 1761, 1762, 1741, 1764, 1765, 1766, 1628, 1771, 1772, 1763, + /* 490 */ 1721, 1768, 1733, 1782, 1724, 1767, 1784, 1729, 1786, 1735, + /* 500 */ 1793, 1795, 1754, 1773, 1756, 1757, 1770, 1783, 1774, 1798, + /* 510 */ 1775, 1779, 1801, 1803, 1805, 1780, 1629, 1807, 1808, 1811, + /* 520 */ 1776, 1813, 1814, 1787, 1777, 1785, 1823, 1790, 1778, 1792, + /* 530 */ 1829, 1797, 1788, 1794, 1834, 1800, 1789, 1802, 1839, 1840, + /* 540 */ 1843, 1844, 1736, 1739, 1816, 1827, 1852, 1818, 1819, 1820, + /* 550 */ 1821, 1822, 1825, 1828, 1832, 1833, 1830, 1838, 1836, 1845, + /* 560 */ 1866, 1853, 1878, 1857, 1846, 1883, 1863, 1869, 1908, 1874, + /* 570 */ 1910, 1876, 1912, 1891, 1894, 1880, 1881, 1884, 1817, 1824, + /* 580 */ 1920, 1796, 1826, 1728, 1890, 1901, 1927, 1740, 1909, 1799, + /* 590 */ 1781, 1930, 1934, 1804, 1791, 1932, 1892, 1681, 1842, 1837, + /* 600 */ 1848, 1895, 1841, 1898, 1854, 1851, 1896, 1911, 1856, 1860, + /* 610 */ 1861, 1862, 1864, 1919, 1906, 1907, 1868, 1925, 1714, 1873, + /* 620 */ 1875, 1972, 1933, 1722, 1945, 1947, 1948, 1959, 1966, 1969, + /* 630 */ 1905, 1921, 1962, 1806, 1976, 1974, 2023, 2024, 2025, 2026, + /* 640 */ 1928, 1988, 1757, 1982, 1931, 1936, 1937, 1939, 1940, 1870, + /* 650 */ 1943, 2032, 1994, 1877, 1949, 1941, 1757, 2000, 2008, 1955, + /* 660 */ 1850, 1956, 2053, 2036, 1847, 1960, 1964, 1961, 1965, 1963, + /* 670 */ 1967, 2022, 1975, 1979, 2033, 1989, 2067, 1889, 1992, 1983, + /* 680 */ 2006, 2060, 2070, 2009, 2007, 2075, 2011, 2012, 2080, 2019, + /* 690 */ 2021, 2085, 2027, 2028, 2086, 2030, 2001, 2002, 2003, 2010, + /* 700 */ 2031, 2082, 2034, 2093, 2037, 2082, 2082, 2111, 2069, 2071, + /* 710 */ 2103, 2105, 2106, 2107, 2108, 2109, 2110, 2112, 2074, 2054, + /* 720 */ 2117, 2128, 2130, 2131, 2132, 2133, 2134, 2135, 2095, 1832, + /* 730 */ 2136, 1833, 2137, 2138, 2140, 2141, 2155, 2143, 2179, 2145, + /* 740 */ 2139, 2142, 2182, 2148, 2147, 2159, 2186, 2164, 2151, 2163, + /* 750 */ 2203, 2169, 2156, 2167, 2207, 2173, 2175, 2211, 2190, 2193, + /* 760 */ 2194, 2195, 2197, 2199, }; -#define YY_REDUCE_COUNT (314) -#define YY_REDUCE_MIN (-397) -#define YY_REDUCE_MAX (2375) +#define YY_REDUCE_COUNT (316) +#define YY_REDUCE_MIN (-445) +#define YY_REDUCE_MAX (2473) static const short yy_reduce_ofst[] = { - /* 0 */ -156, 364, -82, 588, -20, 614, 804, 890, 1024, 1050, - /* 10 */ 1118, 1222, 1288, 1316, 1348, 1419, 174, -333, 154, 450, - /* 20 */ 674, 1447, 1097, 1482, 1466, 1543, 1566, 1660, 1692, 1710, - /* 30 */ 1757, 1804, 1826, 1851, 1873, 1937, 1957, 1972, 2004, 2072, - /* 40 */ 2091, 2119, 2138, 2185, 2242, 2258, 2308, 2328, 2375, 383, - /* 50 */ 270, -397, 83, -395, 371, 416, 554, 75, 501, 235, - /* 60 */ 786, 827, -139, 683, 834, -195, -165, -359, -80, -172, - /* 70 */ -13, -383, -254, -207, -213, 149, -52, 290, -152, -5, - /* 80 */ 263, 376, 407, 413, 453, 82, 508, 593, 462, 271, - /* 90 */ 610, 631, 275, 657, 374, 725, 301, 745, 372, -170, - /* 100 */ 748, 796, 399, 808, -83, 421, 605, 810, 260, -319, - /* 110 */ -338, -338, -293, -327, 94, 237, 433, 502, 534, 669, - /* 120 */ 693, 712, 802, 807, 809, 813, 815, 833, 849, 854, - /* 130 */ 865, 867, 868, -275, -251, -131, 355, 467, -251, 437, - /* 140 */ 559, 452, 445, 479, 726, -334, -85, 719, 681, 168, - /* 150 */ 736, 268, 134, 805, 797, 799, 742, 850, -369, 387, - /* 160 */ 499, 580, 713, 832, 913, 713, 637, 927, 485, 899, - /* 170 */ 818, 835, 949, 839, 941, 945, 940, 940, 980, 939, - /* 180 */ 1032, 1028, 994, 984, 933, 933, 919, 933, 947, 938, - /* 190 */ 940, 974, 976, 996, 1012, 1011, 1018, 1022, 1070, 1071, - /* 200 */ 1026, 1027, 1037, 1063, 1073, 1089, 1083, 1094, 1096, 1099, - /* 210 */ 1123, 1127, 1125, 1130, 1060, 1132, 1100, 1137, 1142, 1090, - /* 220 */ 1144, 1151, 1146, 1147, 1148, 1157, 1153, 1154, 1158, 1169, - /* 230 */ 1138, 1139, 1140, 1141, 1156, 1160, 1161, 1167, 1168, 1175, - /* 240 */ 1177, 1164, 1172, 1131, 1134, 1135, 1110, 1116, 1119, 1187, - /* 250 */ 1145, 1155, 1176, 1201, 1162, 1224, 1179, 1180, 1174, 1108, - /* 260 */ 1171, 1191, 1126, 1184, 1203, 1204, 940, 1128, 1133, 1166, - /* 270 */ 1150, 1170, 1173, 1178, 1122, 1182, 1185, 933, 1251, 1181, - /* 280 */ 1261, 1258, 1263, 1264, 1213, 1212, 1228, 1232, 1235, 1236, - /* 290 */ 1237, 1216, 1241, 1226, 1268, 1269, 1279, 1282, 1192, 1278, - /* 300 */ 1250, 1274, 1291, 1290, 1303, 1306, 1252, 1243, 1244, 1270, - /* 310 */ 1296, 1299, 1300, 1322, 1334, + /* 0 */ -195, 364, 220, 588, 614, 804, 890, 1024, 1050, 1122, + /* 10 */ 1148, 1181, 1258, 1289, 1317, 1352, 153, 674, -332, 1446, + /* 20 */ 1466, 1477, 1563, 1574, 1589, 1660, 1675, 1734, 1769, 1831, + /* 30 */ 1888, 1899, 1918, 1985, 2004, 2020, 2035, 2051, 2119, 2150, + /* 40 */ 2185, 2216, 2282, 2303, 2329, 2348, 2379, 2405, 2473, -301, + /* 50 */ -202, 251, -104, 515, 798, 800, 830, 544, 632, -92, + /* 60 */ 377, 643, 253, 383, 780, -445, -276, -178, -330, -294, + /* 70 */ 45, -221, -337, -242, 115, -203, 175, 206, -306, 50, + /* 80 */ 297, 499, 523, 591, 593, 152, 495, 599, -159, -232, + /* 90 */ 209, 603, 191, 723, 319, 730, 349, 746, 430, 239, + /* 100 */ 797, 802, 366, 808, 413, 578, 681, 817, 564, -333, + /* 110 */ -430, -430, 114, -247, -29, 22, 108, 445, 473, 475, + /* 120 */ 482, 557, 617, 638, 640, 644, 649, 675, 686, 719, + /* 130 */ 722, 760, 822, -44, -20, -8, 211, 371, -20, 338, + /* 140 */ 354, 257, 306, 506, 498, 410, 662, 765, 344, 158, + /* 150 */ 268, 598, 368, 386, 699, 785, 627, 811, -363, 292, + /* 160 */ 573, 596, 624, 732, 739, 624, 847, 828, 921, 892, + /* 170 */ 812, 819, 943, 833, 927, 933, 922, 922, 952, 918, + /* 180 */ 985, 984, 947, 942, 882, 882, 871, 882, 920, 914, + /* 190 */ 922, 969, 956, 971, 993, 994, 1000, 1003, 1054, 1055, + /* 200 */ 1009, 1005, 1012, 1049, 1051, 1063, 1057, 1068, 1069, 1070, + /* 210 */ 1079, 1080, 1082, 1083, 1013, 1073, 1042, 1081, 1084, 1027, + /* 220 */ 1085, 1089, 1086, 1087, 1088, 1097, 1091, 1104, 1106, 1109, + /* 230 */ 1090, 1092, 1099, 1100, 1107, 1110, 1112, 1116, 1117, 1118, + /* 240 */ 1123, 1119, 1144, 1072, 1074, 1076, 1093, 1094, 1098, 1147, + /* 250 */ 1095, 1101, 1115, 1153, 1105, 1120, 1158, 1111, 1124, 1129, + /* 260 */ 1056, 1126, 1135, 1064, 1130, 1137, 1139, 922, 1067, 1071, + /* 270 */ 1075, 1125, 1102, 1132, 1134, 1062, 1108, 1127, 882, 1209, + /* 280 */ 1138, 1166, 1217, 1210, 1227, 1230, 1172, 1185, 1202, 1204, + /* 290 */ 1207, 1208, 1211, 1195, 1216, 1206, 1248, 1213, 1265, 1268, + /* 300 */ 1174, 1243, 1233, 1269, 1282, 1270, 1287, 1290, 1221, 1215, + /* 310 */ 1235, 1236, 1271, 1272, 1274, 1288, 1299, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 10 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 20 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 30 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 40 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 50 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 60 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 70 */ 1719, 1719, 1719, 1719, 1992, 1719, 1719, 1719, 1719, 1719, - /* 80 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1801, 1719, - /* 90 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 100 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1799, 1985, - /* 110 */ 2206, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 120 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 130 */ 1719, 1719, 1719, 1719, 2218, 1719, 1719, 1719, 2218, 2218, - /* 140 */ 2218, 1799, 2178, 2178, 1719, 1719, 1719, 1719, 1801, 2050, - /* 150 */ 1719, 1719, 1719, 1719, 1719, 1719, 1920, 1719, 1719, 1719, - /* 160 */ 1719, 1719, 1944, 1719, 1719, 1719, 2044, 1719, 1719, 2243, - /* 170 */ 2299, 1719, 1719, 2246, 1719, 1719, 1719, 1719, 1719, 1997, - /* 180 */ 1719, 1719, 1874, 2233, 2210, 2224, 2283, 2211, 2208, 2227, - /* 190 */ 1719, 2237, 1719, 2031, 1990, 1719, 1990, 1987, 1719, 1719, - /* 200 */ 1990, 1987, 1987, 1863, 1859, 1719, 1857, 1719, 1719, 1719, - /* 210 */ 1719, 1766, 1719, 1766, 1719, 1801, 1719, 1801, 1719, 1719, - /* 220 */ 1801, 1719, 1801, 1801, 1801, 1719, 1801, 1779, 1779, 1719, - /* 230 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 240 */ 1719, 1719, 1719, 1719, 1719, 1719, 2062, 2048, 1719, 1799, - /* 250 */ 2042, 2040, 1719, 1799, 2038, 1719, 1719, 1719, 1719, 2254, - /* 260 */ 2252, 1719, 2254, 2252, 1719, 1719, 1719, 2268, 2264, 2254, - /* 270 */ 2272, 2270, 2239, 2237, 2302, 2289, 2285, 2224, 1719, 1719, - /* 280 */ 1719, 1719, 1799, 1799, 1719, 2252, 1719, 1719, 1719, 1719, - /* 290 */ 1719, 2252, 1719, 1719, 1799, 1719, 1799, 1719, 1719, 1890, - /* 300 */ 1719, 1719, 1719, 1799, 1751, 1719, 2033, 2053, 2015, 2015, - /* 310 */ 1923, 1923, 1923, 1802, 1724, 1719, 1719, 1719, 1719, 1719, - /* 320 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2267, 2266, 2133, - /* 330 */ 1719, 2182, 2181, 2180, 2171, 2132, 1886, 1719, 2131, 2130, - /* 340 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2006, 2005, - /* 350 */ 2124, 1719, 1719, 2125, 2123, 2122, 1719, 1719, 1719, 1719, - /* 360 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 370 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 380 */ 2286, 2290, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2207, - /* 390 */ 1719, 1719, 1719, 2106, 1719, 1719, 1719, 1719, 1719, 1719, - /* 400 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 410 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 420 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 430 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 440 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 450 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 460 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 470 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 480 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 490 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 500 */ 1719, 1719, 1756, 2111, 1719, 1719, 1719, 1719, 1719, 1719, - /* 510 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 520 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 530 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 540 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 550 */ 1719, 1719, 1840, 1839, 1719, 1719, 1719, 1719, 1719, 1719, - /* 560 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 570 */ 1719, 1719, 1719, 1719, 1719, 1719, 2115, 1719, 1719, 1719, - /* 580 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 590 */ 1719, 1719, 1719, 2282, 2240, 1719, 1719, 1719, 1719, 1719, - /* 600 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 610 */ 1719, 1719, 1719, 2106, 1719, 2265, 1719, 1719, 2280, 1719, - /* 620 */ 2284, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2217, 2213, - /* 630 */ 1719, 1719, 2209, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 640 */ 2114, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 650 */ 1719, 1719, 1719, 1719, 2105, 1719, 2168, 1719, 1719, 1719, - /* 660 */ 2202, 1719, 1719, 2153, 1719, 1719, 1719, 1719, 1719, 1719, - /* 670 */ 1719, 1719, 1719, 2115, 1719, 2118, 1719, 1719, 1719, 1719, - /* 680 */ 1719, 1917, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 690 */ 1719, 1719, 1719, 1719, 1902, 1900, 1899, 1898, 1719, 1930, - /* 700 */ 1719, 1719, 1719, 1926, 1925, 1719, 1719, 1719, 1719, 1719, - /* 710 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1820, 1719, - /* 720 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1812, 1719, 1811, - /* 730 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 740 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 750 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, - /* 760 */ 1719, 1719, + /* 0 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 10 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 20 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 30 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 40 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 50 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 60 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 70 */ 1721, 1721, 1721, 1721, 1994, 1721, 1721, 1721, 1721, 1721, + /* 80 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1803, 1721, + /* 90 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 100 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1801, 1987, + /* 110 */ 2208, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 120 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 130 */ 1721, 1721, 1721, 1721, 2220, 1721, 1721, 1721, 2220, 2220, + /* 140 */ 2220, 1801, 2180, 2180, 1721, 1721, 1721, 1721, 1803, 2052, + /* 150 */ 1721, 1721, 1721, 1721, 1721, 1721, 1922, 1721, 1721, 1721, + /* 160 */ 1721, 1721, 1946, 1721, 1721, 1721, 2046, 1721, 1721, 2245, + /* 170 */ 2301, 1721, 1721, 2248, 1721, 1721, 1721, 1721, 1721, 1999, + /* 180 */ 1721, 1721, 1876, 2235, 2212, 2226, 2285, 2213, 2210, 2229, + /* 190 */ 1721, 2239, 1721, 2033, 1992, 1721, 1992, 1989, 1721, 1721, + /* 200 */ 1992, 1989, 1989, 1865, 1861, 1721, 1859, 1721, 1721, 1721, + /* 210 */ 1721, 1768, 1721, 1768, 1721, 1803, 1721, 1803, 1721, 1721, + /* 220 */ 1803, 1721, 1803, 1803, 1803, 1721, 1803, 1781, 1781, 1721, + /* 230 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 240 */ 1721, 1721, 1721, 1721, 1721, 1721, 2064, 2050, 1721, 1801, + /* 250 */ 2044, 2042, 1721, 1801, 2040, 2239, 1721, 1721, 1721, 1721, + /* 260 */ 2256, 2254, 1721, 2256, 2254, 1721, 1721, 1721, 2270, 2266, + /* 270 */ 2256, 2274, 2272, 2241, 2239, 2304, 2291, 2287, 2226, 1721, + /* 280 */ 1721, 2239, 1721, 1721, 1801, 1801, 1721, 2254, 1721, 1721, + /* 290 */ 1721, 1721, 1721, 2254, 1721, 1721, 1801, 1721, 1801, 1721, + /* 300 */ 1721, 1892, 1721, 1721, 1721, 1801, 1753, 1721, 2035, 2055, + /* 310 */ 2017, 2017, 1925, 1925, 1925, 1804, 1726, 1721, 1721, 1721, + /* 320 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2269, + /* 330 */ 2268, 2135, 1721, 2184, 2183, 2182, 2173, 2134, 1888, 1721, + /* 340 */ 2133, 2132, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 350 */ 2008, 2007, 2126, 1721, 1721, 2127, 2125, 2124, 1721, 1721, + /* 360 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 370 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 380 */ 1721, 1721, 2288, 2292, 1721, 1721, 1721, 1721, 1721, 1721, + /* 390 */ 1721, 2209, 1721, 1721, 1721, 2108, 1721, 1721, 1721, 1721, + /* 400 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 410 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 420 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 430 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 440 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 450 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 460 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 470 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 480 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 490 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 500 */ 1721, 1721, 1721, 1721, 1758, 2113, 1721, 1721, 1721, 1721, + /* 510 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 520 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 530 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 540 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 550 */ 1721, 1721, 1721, 1721, 1842, 1841, 1721, 1721, 1721, 1721, + /* 560 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 570 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2117, 1721, + /* 580 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 590 */ 1721, 1721, 1721, 1721, 1721, 2284, 2242, 1721, 1721, 1721, + /* 600 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 610 */ 1721, 1721, 1721, 1721, 1721, 2108, 1721, 2267, 1721, 1721, + /* 620 */ 2282, 1721, 2286, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 630 */ 2219, 2215, 1721, 1721, 2211, 1721, 1721, 1721, 1721, 1721, + /* 640 */ 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 650 */ 1721, 1721, 1721, 1721, 1721, 1721, 2107, 1721, 2170, 1721, + /* 660 */ 1721, 1721, 2204, 1721, 1721, 2155, 1721, 1721, 1721, 1721, + /* 670 */ 1721, 1721, 1721, 1721, 1721, 2117, 1721, 2120, 1721, 1721, + /* 680 */ 1721, 1721, 1721, 1919, 1721, 1721, 1721, 1721, 1721, 1721, + /* 690 */ 1721, 1721, 1721, 1721, 1721, 1721, 1904, 1902, 1901, 1900, + /* 700 */ 1721, 1932, 1721, 1721, 1721, 1928, 1927, 1721, 1721, 1721, + /* 710 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 720 */ 1822, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1814, + /* 730 */ 1721, 1813, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 740 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 750 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 760 */ 1721, 1721, 1721, 1721, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1873,22 +1873,22 @@ static const char *const yyTokenName[] = { /* 399 */ "func", /* 400 */ "sma_func_name", /* 401 */ "query_or_subquery", - /* 402 */ "cgroup_name", - /* 403 */ "analyze_opt", - /* 404 */ "explain_options", - /* 405 */ "insert_query", - /* 406 */ "or_replace_opt", - /* 407 */ "agg_func_opt", - /* 408 */ "bufsize_opt", - /* 409 */ "language_opt", - /* 410 */ "stream_name", - /* 411 */ "stream_options", - /* 412 */ "col_list_opt", - /* 413 */ "tag_def_or_ref_opt", - /* 414 */ "subtable_opt", - /* 415 */ "expression", - /* 416 */ "dnode_list", - /* 417 */ "where_clause_opt", + /* 402 */ "where_clause_opt", + /* 403 */ "cgroup_name", + /* 404 */ "analyze_opt", + /* 405 */ "explain_options", + /* 406 */ "insert_query", + /* 407 */ "or_replace_opt", + /* 408 */ "agg_func_opt", + /* 409 */ "bufsize_opt", + /* 410 */ "language_opt", + /* 411 */ "stream_name", + /* 412 */ "stream_options", + /* 413 */ "col_list_opt", + /* 414 */ "tag_def_or_ref_opt", + /* 415 */ "subtable_opt", + /* 416 */ "expression", + /* 417 */ "dnode_list", /* 418 */ "literal_func", /* 419 */ "literal_list", /* 420 */ "table_alias", @@ -2254,8 +2254,8 @@ static const char *const yyRuleName[] = { /* 297 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", /* 298 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", /* 299 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 300 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 301 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 300 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt", + /* 301 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt", /* 302 */ "cmd ::= DROP TOPIC exists_opt topic_name", /* 303 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", /* 304 */ "cmd ::= DESC full_table_name", @@ -2695,12 +2695,12 @@ static void yy_destructor( case 398: /* sma_stream_opt */ case 399: /* func */ case 401: /* query_or_subquery */ - case 404: /* explain_options */ - case 405: /* insert_query */ - case 411: /* stream_options */ - case 414: /* subtable_opt */ - case 415: /* expression */ - case 417: /* where_clause_opt */ + case 402: /* where_clause_opt */ + case 405: /* explain_options */ + case 406: /* insert_query */ + case 412: /* stream_options */ + case 415: /* subtable_opt */ + case 416: /* expression */ case 418: /* literal_func */ case 421: /* expr_or_subquery */ case 422: /* pseudo_column */ @@ -2745,7 +2745,7 @@ static void yy_destructor( case 332: /* alter_account_options */ case 334: /* alter_account_option */ case 352: /* speed_opt */ - case 408: /* bufsize_opt */ + case 409: /* bufsize_opt */ { } @@ -2760,9 +2760,9 @@ static void yy_destructor( case 392: /* column_alias */ case 395: /* index_name */ case 400: /* sma_func_name */ - case 402: /* cgroup_name */ - case 409: /* language_opt */ - case 410: /* stream_name */ + case 403: /* cgroup_name */ + case 410: /* language_opt */ + case 411: /* stream_name */ case 420: /* table_alias */ case 426: /* star_func */ case 428: /* noarg_func */ @@ -2791,9 +2791,9 @@ static void yy_destructor( case 347: /* force_opt */ case 348: /* not_exists_opt */ case 350: /* exists_opt */ - case 403: /* analyze_opt */ - case 406: /* or_replace_opt */ - case 407: /* agg_func_opt */ + case 404: /* analyze_opt */ + case 407: /* or_replace_opt */ + case 408: /* agg_func_opt */ case 451: /* set_quantifier_opt */ { @@ -2814,9 +2814,9 @@ static void yy_destructor( case 380: /* rollup_func_list */ case 390: /* tag_list_opt */ case 396: /* func_list */ - case 412: /* col_list_opt */ - case 413: /* tag_def_or_ref_opt */ - case 416: /* dnode_list */ + case 413: /* col_list_opt */ + case 414: /* tag_def_or_ref_opt */ + case 417: /* dnode_list */ case 419: /* literal_list */ case 427: /* star_func_para_list */ case 429: /* other_para_list */ @@ -3455,8 +3455,8 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 330, /* (297) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ 330, /* (298) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ 330, /* (299) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - 330, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - 330, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + 330, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt */ + 330, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt */ 330, /* (302) cmd ::= DROP TOPIC exists_opt topic_name */ 330, /* (303) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ 330, /* (304) cmd ::= DESC full_table_name */ @@ -3464,39 +3464,39 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 330, /* (306) cmd ::= RESET QUERY CACHE */ 330, /* (307) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ 330, /* (308) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ - 403, /* (309) analyze_opt ::= */ - 403, /* (310) analyze_opt ::= ANALYZE */ - 404, /* (311) explain_options ::= */ - 404, /* (312) explain_options ::= explain_options VERBOSE NK_BOOL */ - 404, /* (313) explain_options ::= explain_options RATIO NK_FLOAT */ + 404, /* (309) analyze_opt ::= */ + 404, /* (310) analyze_opt ::= ANALYZE */ + 405, /* (311) explain_options ::= */ + 405, /* (312) explain_options ::= explain_options VERBOSE NK_BOOL */ + 405, /* (313) explain_options ::= explain_options RATIO NK_FLOAT */ 330, /* (314) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ 330, /* (315) cmd ::= DROP FUNCTION exists_opt function_name */ - 407, /* (316) agg_func_opt ::= */ - 407, /* (317) agg_func_opt ::= AGGREGATE */ - 408, /* (318) bufsize_opt ::= */ - 408, /* (319) bufsize_opt ::= BUFSIZE NK_INTEGER */ - 409, /* (320) language_opt ::= */ - 409, /* (321) language_opt ::= LANGUAGE NK_STRING */ - 406, /* (322) or_replace_opt ::= */ - 406, /* (323) or_replace_opt ::= OR REPLACE */ + 408, /* (316) agg_func_opt ::= */ + 408, /* (317) agg_func_opt ::= AGGREGATE */ + 409, /* (318) bufsize_opt ::= */ + 409, /* (319) bufsize_opt ::= BUFSIZE NK_INTEGER */ + 410, /* (320) language_opt ::= */ + 410, /* (321) language_opt ::= LANGUAGE NK_STRING */ + 407, /* (322) or_replace_opt ::= */ + 407, /* (323) or_replace_opt ::= OR REPLACE */ 330, /* (324) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ 330, /* (325) cmd ::= DROP STREAM exists_opt stream_name */ - 412, /* (326) col_list_opt ::= */ - 412, /* (327) col_list_opt ::= NK_LP col_name_list NK_RP */ - 413, /* (328) tag_def_or_ref_opt ::= */ - 413, /* (329) tag_def_or_ref_opt ::= tags_def */ - 413, /* (330) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ - 411, /* (331) stream_options ::= */ - 411, /* (332) stream_options ::= stream_options TRIGGER AT_ONCE */ - 411, /* (333) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - 411, /* (334) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - 411, /* (335) stream_options ::= stream_options WATERMARK duration_literal */ - 411, /* (336) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - 411, /* (337) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - 411, /* (338) stream_options ::= stream_options DELETE_MARK duration_literal */ - 411, /* (339) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - 414, /* (340) subtable_opt ::= */ - 414, /* (341) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + 413, /* (326) col_list_opt ::= */ + 413, /* (327) col_list_opt ::= NK_LP col_name_list NK_RP */ + 414, /* (328) tag_def_or_ref_opt ::= */ + 414, /* (329) tag_def_or_ref_opt ::= tags_def */ + 414, /* (330) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + 412, /* (331) stream_options ::= */ + 412, /* (332) stream_options ::= stream_options TRIGGER AT_ONCE */ + 412, /* (333) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + 412, /* (334) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + 412, /* (335) stream_options ::= stream_options WATERMARK duration_literal */ + 412, /* (336) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + 412, /* (337) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + 412, /* (338) stream_options ::= stream_options DELETE_MARK duration_literal */ + 412, /* (339) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + 415, /* (340) subtable_opt ::= */ + 415, /* (341) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ 330, /* (342) cmd ::= KILL CONNECTION NK_INTEGER */ 330, /* (343) cmd ::= KILL QUERY NK_STRING */ 330, /* (344) cmd ::= KILL TRANSACTION NK_INTEGER */ @@ -3505,13 +3505,13 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 330, /* (347) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ 330, /* (348) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ 330, /* (349) cmd ::= SPLIT VGROUP NK_INTEGER */ - 416, /* (350) dnode_list ::= DNODE NK_INTEGER */ - 416, /* (351) dnode_list ::= dnode_list DNODE NK_INTEGER */ + 417, /* (350) dnode_list ::= DNODE NK_INTEGER */ + 417, /* (351) dnode_list ::= dnode_list DNODE NK_INTEGER */ 330, /* (352) cmd ::= DELETE FROM full_table_name where_clause_opt */ 330, /* (353) cmd ::= query_or_subquery */ 330, /* (354) cmd ::= insert_query */ - 405, /* (355) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - 405, /* (356) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + 406, /* (355) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + 406, /* (356) insert_query ::= INSERT INTO full_table_name query_or_subquery */ 333, /* (357) literal ::= NK_INTEGER */ 333, /* (358) literal ::= NK_FLOAT */ 333, /* (359) literal ::= NK_STRING */ @@ -3545,26 +3545,26 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 392, /* (387) column_alias ::= NK_ID */ 335, /* (388) user_name ::= NK_ID */ 344, /* (389) topic_name ::= NK_ID */ - 410, /* (390) stream_name ::= NK_ID */ - 402, /* (391) cgroup_name ::= NK_ID */ + 411, /* (390) stream_name ::= NK_ID */ + 403, /* (391) cgroup_name ::= NK_ID */ 395, /* (392) index_name ::= NK_ID */ 421, /* (393) expr_or_subquery ::= expression */ - 415, /* (394) expression ::= literal */ - 415, /* (395) expression ::= pseudo_column */ - 415, /* (396) expression ::= column_reference */ - 415, /* (397) expression ::= function_expression */ - 415, /* (398) expression ::= case_when_expression */ - 415, /* (399) expression ::= NK_LP expression NK_RP */ - 415, /* (400) expression ::= NK_PLUS expr_or_subquery */ - 415, /* (401) expression ::= NK_MINUS expr_or_subquery */ - 415, /* (402) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - 415, /* (403) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - 415, /* (404) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - 415, /* (405) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - 415, /* (406) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - 415, /* (407) expression ::= column_reference NK_ARROW NK_STRING */ - 415, /* (408) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - 415, /* (409) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + 416, /* (394) expression ::= literal */ + 416, /* (395) expression ::= pseudo_column */ + 416, /* (396) expression ::= column_reference */ + 416, /* (397) expression ::= function_expression */ + 416, /* (398) expression ::= case_when_expression */ + 416, /* (399) expression ::= NK_LP expression NK_RP */ + 416, /* (400) expression ::= NK_PLUS expr_or_subquery */ + 416, /* (401) expression ::= NK_MINUS expr_or_subquery */ + 416, /* (402) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + 416, /* (403) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + 416, /* (404) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + 416, /* (405) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + 416, /* (406) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + 416, /* (407) expression ::= column_reference NK_ARROW NK_STRING */ + 416, /* (408) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + 416, /* (409) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ 375, /* (410) expression_list ::= expr_or_subquery */ 375, /* (411) expression_list ::= expression_list NK_COMMA expr_or_subquery */ 423, /* (412) column_reference ::= column_name */ @@ -3670,8 +3670,8 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 460, /* (512) select_item ::= common_expression column_alias */ 460, /* (513) select_item ::= common_expression AS column_alias */ 460, /* (514) select_item ::= table_name NK_DOT NK_STAR */ - 417, /* (515) where_clause_opt ::= */ - 417, /* (516) where_clause_opt ::= WHERE search_condition */ + 402, /* (515) where_clause_opt ::= */ + 402, /* (516) where_clause_opt ::= WHERE search_condition */ 453, /* (517) partition_by_clause_opt ::= */ 453, /* (518) partition_by_clause_opt ::= PARTITION BY partition_list */ 461, /* (519) partition_list ::= partition_item */ @@ -4043,8 +4043,8 @@ static const signed char yyRuleInfoNRhs[] = { -6, /* (297) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -7, /* (298) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -9, /* (299) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - -7, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - -9, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + -8, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt */ + -10, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt */ -4, /* (302) cmd ::= DROP TOPIC exists_opt topic_name */ -7, /* (303) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -2, /* (304) cmd ::= DESC full_table_name */ @@ -5390,11 +5390,11 @@ static YYACTIONTYPE yy_reduce( case 299: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy369, &yymsp[-5].minor.yy113, &yymsp[0].minor.yy113, true); } break; - case 300: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy369, &yymsp[-3].minor.yy113, yymsp[0].minor.yy448, false); } + case 300: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name where_clause_opt */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-5].minor.yy369, &yymsp[-4].minor.yy113, yymsp[-1].minor.yy448, false, yymsp[0].minor.yy448); } break; - case 301: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy369, &yymsp[-5].minor.yy113, yymsp[0].minor.yy448, true); } + case 301: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name where_clause_opt */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-7].minor.yy369, &yymsp[-6].minor.yy113, yymsp[-1].minor.yy448, true, yymsp[0].minor.yy448); } break; case 302: /* cmd ::= DROP TOPIC exists_opt topic_name */ { pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 6a08193a39..a4e8bdd87a 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -1145,6 +1145,15 @@ TEST_F(ParserInitialCTest, createTopic) { setCreateTopicReq("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1); run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1"); clearCreateTopicReq(); + + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 as stable st1 where tag1 > 0", nullptr, "test", "st1"); + run("CREATE TOPIC IF NOT EXISTS tp1 AS STABLE st1 WHERE tag1 > 0"); + clearCreateTopicReq(); + + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 with meta as stable st1 where tag1 > 0", nullptr, "test", "st1", 1); + run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1 WHERE tag1 > 0"); + clearCreateTopicReq(); + } /* diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 9d8c170003..1260ea336c 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -449,6 +449,18 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } +void getColumnTypeFromMeta(STableMeta* pMeta, char* pName, ETableColumnType* pType) { + int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; + for (int32_t i = 0; i < nums; ++i) { + if (0 == strcmp(pName, pMeta->schema[i].name)) { + *pType = (i < pMeta->tableInfo.numOfColumns) ? TCOL_TYPE_COLUMN : TCOL_TYPE_TAG; + return; + } + } + + *pType = TCOL_TYPE_NONE; +} + void freeVgInfo(SDBVgInfo* vgInfo) { if (NULL == vgInfo) { return; diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index 0bf0873e9f..85a23055a3 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -108,4 +108,14 @@ if $rows != 6 then return -1 endi +return -1 +sql create topic topic_stable_1 as stable stb where t1 > 0 +sql create topic topic_stable_1 as stable stb where t1 > 0 and t1 < 0 +sql create topic topic_stable_1 as stable stb where 1 > 0 +sql create topic topic_stable_1 as stable stb where last(t1) > 0 +sql create topic topic_stable_1 as stable stb where tbname is not null +sql create topic topic_stable_1 as stable stb where tbname > 'a' +sql create topic topic_stable_1 as stable stb where tbname > 0 and xx < 0 +sql create topic topic_stable_1 as stable stb where tbname > 0 and c1 < 0 + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 52c384c7c9db31776dc5c449d2e3c6313e1a1e57 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 26 May 2023 09:06:19 +0800 Subject: [PATCH 099/238] change parameter --- source/libs/stream/src/streamBackendRocksdb.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 11e33da5a4..c530171eb2 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -675,7 +675,6 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { rocksdb_readoptions_destroy(inst->rOpt); taosMemoryFree(inst->cfOpt); - taosMemoryFree(inst->param); taosMemoryFreeClear(inst->param); taosMemoryFree(inst); } From 409656f7b6c81aa61aee935ba403825135565c9d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 10:05:54 +0800 Subject: [PATCH 100/238] fix(query): fix bug in iterating the table list. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5d970ce6c3..2eff4ff8da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3433,6 +3433,8 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { if (!hasNexTable) { return TSDB_CODE_SUCCESS; } + + pBlockScanInfo = pStatus->pTableIter; } initMemDataIterator(*pBlockScanInfo, pReader); From 74f753606b69019d5cd9d415551c3294bb51dfde Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 10:37:36 +0800 Subject: [PATCH 101/238] fix(query): add regression test case. --- tests/script/tsim/compute/last_row.sim | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/script/tsim/compute/last_row.sim b/tests/script/tsim/compute/last_row.sim index 2e060dc285..8e62fbffb5 100644 --- a/tests/script/tsim/compute/last_row.sim +++ b/tests/script/tsim/compute/last_row.sim @@ -213,4 +213,54 @@ if $rows != 2 then return -1 endi +print =======================> regresss bug in last_row query +sql drop database if exists db; +sql create database if not exists db vgroups 1 cachemodel 'both'; +sql create table db.stb (ts timestamp, c0 bigint) tags(t1 int); +sql insert into db.stb_0 using db.stb tags(1) values ('2023-11-23 19:06:40.000', 491173569); +sql insert into db.stb_2 using db.stb tags(3) values ('2023-11-25 19:30:00.000', 2080726142); +sql insert into db.stb_3 using db.stb tags(4) values ('2023-11-26 06:48:20.000', 1907405128); +sql insert into db.stb_4 using db.stb tags(5) values ('2023-11-24 22:56:40.000', 220783803); + +sql create table db.stb_1 using db.stb tags(2); +sql insert into db.stb_1 (ts) values('2023-11-26 13:11:40.000'); +sql insert into db.stb_1 (ts, c0) values('2023-11-26 13:11:39.000', 11); + +sql select tbname,ts,last_row(c0) from db.stb; +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data01 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +sql alter database db cachemodel 'none'; +sql reset query cache; +sql select tbname,last_row(c0, ts) from db.stb; + +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data02 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data01 != NULL then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From ee4709bb4b8c30ba2fe9f997496628b0d169d8c6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 26 May 2023 16:01:42 +0800 Subject: [PATCH 102/238] chore: update docs about support row/columns width up to 64KB --- docs/en/12-taos-sql/01-data-type.md | 2 +- docs/en/12-taos-sql/03-table.md | 2 +- docs/en/12-taos-sql/19-limit.md | 2 +- docs/en/14-reference/13-schemaless/13-schemaless.md | 2 +- docs/zh/12-taos-sql/01-data-type.md | 4 ++-- docs/zh/12-taos-sql/03-table.md | 2 +- docs/zh/12-taos-sql/19-limit.md | 2 +- docs/zh/14-reference/13-schemaless/13-schemaless.md | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/en/12-taos-sql/01-data-type.md b/docs/en/12-taos-sql/01-data-type.md index 70bea97dba..13007d5bb1 100644 --- a/docs/en/12-taos-sql/01-data-type.md +++ b/docs/en/12-taos-sql/01-data-type.md @@ -45,7 +45,7 @@ In TDengine, the data types below can be used when specifying a column or tag. :::note - Only ASCII visible characters are suggested to be used in a column or tag of BINARY type. Multi-byte characters must be stored in NCHAR type. -- The length of BINARY can be up to 16,374 bytes. The string value must be quoted with single quotes. You must specify a length in bytes for a BINARY value, for example binary(20) for up to twenty single-byte characters. If the data exceeds the specified length, an error will occur. The literal single quote inside the string must be preceded with back slash like `\'` +- The length of BINARY can be up to 16,374(data column is 65,517 and tag column is 16,382 since version 3.0.5.0) bytes. The string value must be quoted with single quotes. You must specify a length in bytes for a BINARY value, for example binary(20) for up to twenty single-byte characters. If the data exceeds the specified length, an error will occur. The literal single quote inside the string must be preceded with back slash like `\'` - Numeric values in SQL statements will be determined as integer or float type according to whether there is decimal point or whether scientific notation is used, so attention must be paid to avoid overflow. For example, 9999999999999999999 will be considered as overflow because it exceeds the upper limit of long integer, but 9999999999999999999.0 will be considered as a legal float number. ::: diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index f61d1f5147..7f39fb5867 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -45,7 +45,7 @@ table_option: { 1. The first column of a table MUST be of type TIMESTAMP. It is automatically set as the primary key. 2. The maximum length of the table name is 192 bytes. -3. The maximum length of each row is 48k bytes, please note that the extra 2 bytes used by each BINARY/NCHAR column are also counted. +3. The maximum length of each row is 48k(64k since version 3.0.5.0) bytes, please note that the extra 2 bytes used by each BINARY/NCHAR column are also counted. 4. The name of the subtable can only consist of characters from the English alphabet, digits and underscore. Table names can't start with a digit. Table names are case insensitive. 5. The maximum length in bytes must be specified when using BINARY or NCHAR types. 6. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive. diff --git a/docs/en/12-taos-sql/19-limit.md b/docs/en/12-taos-sql/19-limit.md index 654fae7560..22ad2055e4 100644 --- a/docs/en/12-taos-sql/19-limit.md +++ b/docs/en/12-taos-sql/19-limit.md @@ -26,7 +26,7 @@ The following characters cannot occur in a password: single quotation marks ('), - Maximum length of database name is 64 bytes - Maximum length of table name is 192 bytes, excluding the database name prefix and the separator. -- Maximum length of each data row is 48K bytes. Note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. +- Maximum length of each data row is 48K(64K since version 3.0.5.0) bytes. Note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. - The maximum length of a column name is 64 bytes. - Maximum number of columns is 4096. There must be at least 2 columns, and the first column must be timestamp. - The maximum length of a tag name is 64 bytes diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index aad0e63a42..3ae9098a73 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -90,7 +90,7 @@ You can configure smlChildTableName in taos.cfg to specify table names, for exam Note: TDengine 3.0.3.0 and later automatically detect whether order is consistent. This parameter is no longer used. :::tip -All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. +All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB(64 KB since version 3.0.5.0) and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. ::: ## Time resolution recognition diff --git a/docs/zh/12-taos-sql/01-data-type.md b/docs/zh/12-taos-sql/01-data-type.md index f014573ca6..4a4c1d6ec6 100644 --- a/docs/zh/12-taos-sql/01-data-type.md +++ b/docs/zh/12-taos-sql/01-data-type.md @@ -45,9 +45,9 @@ CREATE DATABASE db_name PRECISION 'ns'; :::note -- 表的每行长度不能超过 48KB（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）。 +- 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）。 - 虽然 BINARY 类型在底层存储上支持字节型的二进制字符，但不同编程语言对二进制数据的处理方式并不保证一致，因此建议在 BINARY 类型中只存储 ASCII 可见字符，而避免存储不可见字符。多字节的数据，例如中文字符，则需要使用 NCHAR 类型进行保存。如果强行使用 BINARY 类型保存中文字符，虽然有时也能正常读写，但并不带有字符集信息，很容易出现数据乱码甚至数据损坏等情况。 -- BINARY 类型理论上最长可以有 16,374 字节。BINARY 仅支持字符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 BINARY(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 字节的存储空间，总共固定占用 20 字节的空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\'`。 +- BINARY 类型理论上最长可以有 16,374（从 3.0.5.0 版本开始，数据列为 65,517，标签列为 16,382） 字节。BINARY 仅支持字符串输入，字符串两端需使用单引号引用。使用时须指定大小，如 BINARY(20) 定义了最长为 20 个单字节字符的字符串，每个字符占 1 字节的存储空间，总共固定占用 20 字节的空间，此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号，可以用转义字符反斜线加单引号来表示，即 `\'`。 - SQL 语句中的数值类型将依据是否存在小数点，或使用科学计数法表示，来判断数值类型是否为整型或者浮点型，因此在使用时要注意相应类型越界的情况。例如，9999999999999999999 会认为超过长整型的上边界而溢出，而 9999999999999999999.0 会被认为是有效的浮点数。 ::: diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 5687c7e740..2e66ac4002 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -43,7 +43,7 @@ table_option: { 1. 表的第一个字段必须是 TIMESTAMP，并且系统自动将其设为主键； 2. 表名最大长度为 192； -3. 表的每行长度不能超过 48KB;（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） +3. 表的每行长度不能超过 48KB（从 3.0.5.0 版本开始为 64KB）;（注意：每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） 4. 子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写 5. 使用数据类型 binary 或 nchar，需指定其最长的字节数，如 binary(20)，表示 20 字节； 6. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 diff --git a/docs/zh/12-taos-sql/19-limit.md b/docs/zh/12-taos-sql/19-limit.md index 7b6692f1b7..e5a492580e 100644 --- a/docs/zh/12-taos-sql/19-limit.md +++ b/docs/zh/12-taos-sql/19-limit.md @@ -26,7 +26,7 @@ description: 合法字符集和命名中的限制规则 - 数据库名最大长度为 64 字节 - 表名最大长度为 192 字节，不包括数据库名前缀和分隔符 -- 每行数据最大长度 48KB （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） +- 每行数据最大长度 48KB（从 3.0.5.0 版本开始为 64KB） （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） - 列名最大长度为 64 字节 - 最多允许 4096 列，最少需要 2 列，第一列必须是时间戳。 - 标签名最大长度为 64 字节 diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index e5f232c1fc..6c2007938b 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -87,7 +87,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 :::tip 无模式所有的处理逻辑，仍会遵循 TDengine 对数据结构的底层限制，例如每行数据的总长度不能超过 -48KB，标签值的总长度不超过16KB。这方面的具体限制约束请参见 [TDengine SQL 边界限制](/taos-sql/limit) +48KB（从 3.0.5.0 版本开始为 64KB），标签值的总长度不超过16KB。这方面的具体限制约束请参见 [TDengine SQL 边界限制](/taos-sql/limit) ::: From faaa9da88748d7b13d645bc2ca3ec42e92bd4359 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 26 May 2023 18:17:05 +0800 Subject: [PATCH 103/238] fix: sliding can't be 1a issue --- source/common/src/tglobal.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bef63d8a49..2a9c54c189 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -139,8 +139,8 @@ int32_t tsCompressColData = -1; // count/hyperloglog function always return values in case of all NULL data or Empty data set. int32_t tsCountAlwaysReturnValue = 1; -// 10 ms for sliding time, the value will changed in case of time precision changed -int32_t tsMinSlidingTime = 10; +// 1 ms for sliding time, the value will changed in case of time precision changed +int32_t tsMinSlidingTime = 1; // the maxinum number of distict query result int32_t tsMaxNumOfDistinctResults = 1000 * 10000; @@ -412,7 +412,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; From 24937c3d5bb9baf6cb90472c7db55c4f03db3cda Mon Sep 17 00:00:00 2001 From: dmchen Date: Fri, 26 May 2023 19:01:13 +0800 Subject: [PATCH 104/238] check empty dnode --- source/dnode/mnode/impl/src/mndDnode.c | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 99694ae4b8..73dbb243a1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -986,20 +986,20 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); - if (numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) { - bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); - if (isonline && force) { - terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; - mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), - numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); - goto _OVER; - } - if (!isonline && !force) { - terrno = TSDB_CODE_DNODE_OFFLINE; - mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), - numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); - goto _OVER; - } + bool isonline = mndIsDnodeOnline(pDnode, taosGetTimestampMs()); + + if (isonline && force) { + terrno = TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE; + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); + goto _OVER; + } + + if (!isonline && !force) { + terrno = TSDB_CODE_DNODE_OFFLINE; + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); + goto _OVER; } code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes, force, dropReq.unsafe); From 50487255b33a79f2d63f6009db00426d8e799c88 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 26 May 2023 19:02:40 +0800 Subject: [PATCH 105/238] fix:put poll to push manager if wal not exist when offset is latest --- source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/tq/tqUtil.c | 61 ++++++----------------------- source/libs/executor/src/executor.c | 3 ++ 3 files changed, 16 insertions(+), 49 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ef36b8429a..e6ba5d8636 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -106,7 +106,6 @@ typedef struct { // STqPushHandle pushHandle; // push STqExecHandle execHandle; // exec SRpcMsg* msg; - int32_t noDataPollCnt; tq_handle_status status; } STqHandle; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 622cc5b049..33d2a238c5 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -16,7 +16,6 @@ #include "tq.h" #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) -#define NO_POLL_CNT 5 static int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp); @@ -88,15 +87,12 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { return 0; } -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, bool* pBlockReturned) { +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest) { uint64_t consumerId = pRequest->consumerId; STqOffsetVal reqOffset = pRequest->reqOffset; STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); int32_t vgId = TD_VID(pTq->pVnode); - *pBlockReturned = false; - // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. if (pOffset != NULL) { *pOffsetVal = pOffset->val; @@ -129,28 +125,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); - - tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.rspOffset.version); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); - tDeleteSMqDataRsp(&dataRsp); - - *pBlockReturned = true; - return code; - } else { - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); - tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - - *pBlockReturned = true; - return code; - } + // offset set to previous version when init + tqOffsetResetToLog(pOffsetVal, walGetLastVer(pTq->pVnode->pWal)); } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); @@ -173,25 +149,19 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); code = tqScanData(pTq, pHandle, &dataRsp, pOffset); - if(code != 0) { + if(code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { - if(pHandle->noDataPollCnt >= NO_POLL_CNT){ // send poll result to client if no data 5 times to avoid lost data - pHandle->noDataPollCnt = 0; - // lock - taosWLockLatch(&pTq->lock); - code = tqRegisterPushHandle(pTq, pHandle, pMsg); - taosWUnLockLatch(&pTq->lock); - tDeleteSMqDataRsp(&dataRsp); - return code; - } - else{ - pHandle->noDataPollCnt++; - } + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST || (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && + dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId)) { + // lock + taosWLockLatch(&pTq->lock); + code = tqRegisterPushHandle(pTq, pHandle, pMsg); + taosWUnLockLatch(&pTq->lock); + tDeleteSMqDataRsp(&dataRsp); + return code; } // NOTE: this pHandle->consumerId may have been changed already. @@ -326,16 +296,11 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ // 1. reset the offset if needed if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { // handle the reset offset cases, according to the consumer's choice. - bool blockReturned = false; - code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest); if (code != 0) { return code; } - // empty block returned, quit - if (blockReturned) { - return 0; - } } else { // use the consumer specified offset // the offset value can not be monotonious increase?? offset = reqOffset; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a73deffa52..fb0b9957f8 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1080,6 +1080,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume + char buf[80] = {0}; + tFormatOffset(buf, 80, &pTaskInfo->streamInfo.currentOffset); + qDebug("currentOffset:%s", buf); if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } From a3e214b9e8eb24b842ad22b96f09e83bcf9e2632 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 26 May 2023 19:17:04 +0800 Subject: [PATCH 106/238] fix:put poll to push manager if wal not exist when offset is latest --- include/libs/wal/wal.h | 3 ++- source/dnode/vnode/src/tq/tqUtil.c | 12 +++--------- source/libs/executor/src/executor.c | 3 --- source/libs/wal/src/walRef.c | 17 +++++++++-------- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 7e106eefde..7af218b78e 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -205,7 +205,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead); int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead); int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead); -SWalRef *walRefFirstVer(SWal *, SWalRef *); +void walRefFirstVer(SWal *, SWalRef *); +void walRefLastVer(SWal *, SWalRef *); SWalRef *walRefCommittedVer(SWal *); SWalRef *walOpenRef(SWal *); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 33d2a238c5..5eaf7b240b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -115,18 +115,12 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand tqOffsetResetToData(pOffsetVal, 0, 0); } } else { - pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); - if (pHandle->pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // offset set to previous version when init + walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - // offset set to previous version when init - tqOffsetResetToLog(pOffsetVal, walGetLastVer(pTq->pVnode->pWal)); + walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); + tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fb0b9957f8..a73deffa52 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1080,9 +1080,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume - char buf[80] = {0}; - tFormatOffset(buf, 80, &pTaskInfo->streamInfo.currentOffset); - qDebug("currentOffset:%s", buf); if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index 6aba661926..eb36389f1d 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -63,21 +63,22 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { return 0; } -SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) { - if (pRef == NULL) { - pRef = walOpenRef(pWal); - if (pRef == NULL) { - return NULL; - } - } +void walRefFirstVer(SWal *pWal, SWalRef *pRef) { taosThreadMutexLock(&pWal->mutex); int64_t ver = walGetFirstVer(pWal); pRef->refVer = ver; taosThreadMutexUnlock(&pWal->mutex); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver); +} - return pRef; +void walRefLastVer(SWal *pWal, SWalRef *pRef) { + taosThreadMutexLock(&pWal->mutex); + int64_t ver = walGetLastVer(pWal); + pRef->refVer = ver; + + taosThreadMutexUnlock(&pWal->mutex); + wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, ver); } SWalRef *walRefCommittedVer(SWal *pWal) { From 34d21bc03fc14d959cc869690f1441ad0b14d4f7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 26 May 2023 20:11:33 +0800 Subject: [PATCH 107/238] change parameter --- cmake/cmake.define | 2 +- include/libs/stream/streamState.h | 3 +- source/libs/stream/src/streamBackendRocksdb.c | 260 +++++++++--------- 3 files changed, 129 insertions(+), 136 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index a233834fbe..35abe512f7 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0) -set(CMAKE_VERBOSE_MAKEFILE ON) +set(CMAKE_VERBOSE_MAKEFILE OFF) set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 1cc61ec072..66e8b7b2a3 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -44,9 +44,10 @@ typedef struct STdbState { void* param; void* env; SListNode* pComparNode; - void* pBackendHandle; + void* pBackend; char idstr[64]; void* compactFactory; + TdThreadRwlock rwLock; TDB* db; TTB* pStateDb; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c530171eb2..37dd7f6297 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -22,6 +22,9 @@ typedef struct SCompactFilteFactory { void* status; } SCompactFilteFactory; +typedef struct { + void* tableOpt; +} RocksdbCfParam; typedef struct { rocksdb_t* db; rocksdb_column_family_handle_t** pHandle; @@ -29,12 +32,13 @@ typedef struct { rocksdb_readoptions_t* rOpt; rocksdb_options_t** cfOpt; rocksdb_options_t* dbOpt; - void* param; - void* pBackendHandle; + RocksdbCfParam* param; + void* pBackend; SListNode* pCompareNode; + rocksdb_comparator_t** pCompares; } RocksdbCfInst; -int32_t streamStateOpenBackendCf(void* backend, char* name, SHashObj* ids); +int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); void destroyRocksdbCfInst(RocksdbCfInst* inst); @@ -46,9 +50,6 @@ unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, c char** newval, size_t* newvlen, unsigned char* value_changed); rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_compactionfiltercontext_t* ctx); -typedef struct { - void* tableOpt; -} RocksdbCfParam; const char* cfName[] = {"default", "state", "fill", "sess", "func", "parname", "partag"}; typedef int (*EncodeFunc)(void* key, char* buf); @@ -114,25 +115,7 @@ void* streamBackendInit(const char* path) { /* list all cf and get prefix */ - int64_t streamId; - int32_t taskId, dummpy = 0; - SHashObj* tbl = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (size_t i = 0; i < nCf; i++) { - char* cf = cfs[i]; - char suffix[64] = {0}; - if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, suffix)) { - char idstr[128] = {0}; - sprintf(idstr, "0x%" PRIx64 "-%d", streamId, taskId); - // qError("make cf name %s", idstr); - if (taosHashGet(tbl, idstr, strlen(idstr) + 1) == NULL) { - taosHashPut(tbl, idstr, strlen(idstr) + 1, &dummpy, sizeof(dummpy)); - } - } else { - continue; - } - } - streamStateOpenBackendCf(pHandle, (char*)path, tbl); - taosHashCleanup(tbl); + streamStateOpenBackendCf(pHandle, (char*)path, cfs, nCf); } rocksdb_list_column_families_destroy(cfs, nCf); @@ -209,7 +192,7 @@ void streamBackendDelCompare(void* backend, void* arg) { } void streamStateDestroy_rocksdb(SStreamState* pState, bool remove) { streamStateCloseBackend(pState, remove); } static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); -int streamGetInit(const char* funcName); +int streamGetInit(SStreamState* pState, const char* funcName); // |key|-----value------| // |key|ttl|len|userData| @@ -679,61 +662,44 @@ void destroyRocksdbCfInst(RocksdbCfInst* inst) { taosMemoryFree(inst); } -int32_t streamStateOpenBackendCf(void* backend, char* name, SHashObj* ids) { +int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf) { SBackendHandle* handle = backend; char* err = NULL; - size_t nSize = taosHashGetSize(ids); - int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); + int64_t streamId; + int32_t taskId, dummy = 0; + char suffix[64] = {0}; + SHashObj* instTbl = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - char** cfNames = taosMemoryCalloc(nSize * cfLen + 1, sizeof(char*)); - void* pIter = taosHashIterate(ids, NULL); - size_t keyLen = 0; - char* idstr = taosHashGetKey(pIter, &keyLen); - for (int i = 0; i < nSize * cfLen + 1; i++) { - cfNames[i] = (char*)taosMemoryCalloc(1, 128); - if (i == 0) { - memcpy(cfNames[0], "default", strlen("default")); - continue; - } + rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); + RocksdbCfParam* params = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam*)); + rocksdb_comparator_t** pCompare = taosMemoryCalloc(nCf, sizeof(rocksdb_comparator_t**)); + rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(nCf, sizeof(rocksdb_column_family_handle_t*)); - GEN_COLUMN_FAMILY_NAME(cfNames[i], idstr, ginitDict[(i - 1) % (cfLen)].key); - if (i % cfLen == 0) { - pIter = taosHashIterate(ids, pIter); - if (pIter != NULL) idstr = taosHashGetKey(pIter, &keyLen); - } - } - rocksdb_options_t** cfOpts = taosMemoryCalloc(nSize * cfLen + 1, sizeof(rocksdb_options_t*)); - RocksdbCfParam* params = taosMemoryCalloc(nSize * cfLen + 1, sizeof(RocksdbCfParam*)); - for (int i = 0; i < nSize * cfLen + 1; i++) { + for (int i = 0; i < nCf; i++) { + char* cf = cfs[i]; + char funcname[64] = {0}; cfOpts[i] = rocksdb_options_create_copy(handle->dbOpt); - if (i == 0) { - continue; + if (i == 0) continue; + if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, funcname)) { + rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); + rocksdb_block_based_options_set_block_cache(tableOpt, handle->cache); + + rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); + rocksdb_block_based_options_set_filter_policy(tableOpt, filter); + + rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)cfOpts[i], tableOpt); + params[i].tableOpt = tableOpt; + + int idx = streamGetInit(NULL, funcname); + SCfInit* cfPara = &ginitDict[idx]; + + rocksdb_comparator_t* compare = + rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_options_set_comparator((rocksdb_options_t*)cfOpts[i], compare); + pCompare[i] = compare; } - // refactor later - rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); - rocksdb_block_based_options_set_block_cache(tableOpt, handle->cache); - - rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); - rocksdb_block_based_options_set_filter_policy(tableOpt, filter); - - rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)cfOpts[i], tableOpt); - params[i].tableOpt = tableOpt; - }; - - rocksdb_comparator_t** pCompare = taosMemoryCalloc(nSize * cfLen + 1, sizeof(rocksdb_comparator_t**)); - for (int i = 0; i < nSize * cfLen + 1; i++) { - if (i == 0) { - continue; - } - SCfInit* cf = &ginitDict[(i - 1) % cfLen]; - - rocksdb_comparator_t* compare = rocksdb_comparator_create(NULL, cf->detroyFunc, cf->cmpFunc, cf->cmpName); - rocksdb_options_set_comparator((rocksdb_options_t*)cfOpts[i], compare); - pCompare[i] = compare; } - rocksdb_column_family_handle_t** cfHandle = - taosMemoryCalloc(nSize * cfLen + 1, sizeof(rocksdb_column_family_handle_t*)); - rocksdb_t* db = rocksdb_open_column_families(handle->dbOpt, name, nSize * cfLen + 1, (const char* const*)cfNames, + rocksdb_t* db = rocksdb_open_column_families(handle->dbOpt, name, nCf, (const char* const*)cfs, (const rocksdb_options_t* const*)cfOpts, cfHandle, &err); if (err != NULL) { qError("failed to open rocksdb cf, reason:%s", err); @@ -742,50 +708,53 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, SHashObj* ids) { qDebug("succ to open rocksdb cf, reason"); } - pIter = taosHashIterate(ids, NULL); - idstr = taosHashGetKey(pIter, &keyLen); - for (int i = 0; i < nSize; i++) { - RocksdbCfInst* inst = taosMemoryCalloc(1, sizeof(RocksdbCfInst)); - rocksdb_column_family_handle_t** subCf = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); - rocksdb_comparator_t** subCompare = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t*)); - RocksdbCfParam* subParam = taosMemoryCalloc(cfLen, sizeof(RocksdbCfParam)); - rocksdb_options_t** subOpt = taosMemoryCalloc(cfLen, sizeof(rocksdb_options_t*)); - for (int j = 0; j < cfLen; j++) { - subCf[j] = cfHandle[i * cfLen + j + 1]; - subCompare[j] = pCompare[i * cfLen + j + 1]; - subParam[j] = params[i * cfLen + j + 1]; - subOpt[j] = cfOpts[i * cfLen + j + 1]; + static int32_t cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); + for (int i = 0; i < nCf; i++) { + char* cf = cfs[i]; + if (i == 0) continue; + char funcname[64] = {0}; + if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, funcname)) { + char idstr[128] = {0}; + sprintf(idstr, "0x%" PRIx64 "-%d", streamId, taskId); + + int idx = streamGetInit(NULL, funcname); + + RocksdbCfInst* inst = NULL; + RocksdbCfInst** pInst = taosHashGet(instTbl, idstr, strlen(idstr) + 1); + if (pInst == NULL || *pInst == NULL) { + inst = taosMemoryCalloc(1, sizeof(RocksdbCfInst)); + inst->pHandle = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); + inst->cfOpt = taosMemoryCalloc(cfLen, sizeof(rocksdb_options_t*)); + inst->wOpt = rocksdb_writeoptions_create(); + inst->rOpt = rocksdb_readoptions_create(); + inst->param = taosMemoryCalloc(cfLen, sizeof(RocksdbCfParam)); + inst->pBackend = handle; + inst->db = db; + inst->pCompares = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t*)); + + inst->dbOpt = handle->dbOpt; + rocksdb_writeoptions_disable_WAL(inst->wOpt, 1); + taosHashPut(handle->cfInst, idstr, strlen(idstr) + 1, &inst, sizeof(void*)); + } else { + inst = *pInst; + } + inst->cfOpt[idx] = cfOpts[i]; + inst->pCompares[idx] = pCompare[i]; + memcpy(&(inst->param[idx]), &(params[i]), sizeof(RocksdbCfParam)); + inst->pHandle[idx] = cfHandle[i]; } - inst->db = db; - inst->pHandle = subCf; - inst->wOpt = rocksdb_writeoptions_create(); - inst->rOpt = rocksdb_readoptions_create(); - inst->cfOpt = (rocksdb_options_t**)subOpt; - inst->dbOpt = handle->dbOpt; - inst->param = subParam; - inst->pBackendHandle = handle; - handle->db = db; - SCfComparator compare = {.comp = subCompare, .numOfComp = cfLen}; + } + void** pIter = taosHashIterate(handle->cfInst, NULL); + while (*pIter) { + RocksdbCfInst* inst = *pIter; + SCfComparator compare = {.comp = inst->pCompares, .numOfComp = cfLen}; inst->pCompareNode = streamBackendAddCompare(handle, &compare); - rocksdb_writeoptions_disable_WAL(inst->wOpt, 1); - - taosHashPut(handle->cfInst, idstr, keyLen, &inst, sizeof(void*)); - - pIter = taosHashIterate(ids, pIter); - if (pIter != NULL) idstr = taosHashGetKey(pIter, &keyLen); } - rocksdb_column_family_handle_destroy(cfHandle[0]); - rocksdb_options_destroy(cfOpts[0]); - for (int i = 0; i < nSize * cfLen + 1; i++) { - taosMemoryFree(cfNames[i]); - } - taosMemoryFree(cfNames); taosMemoryFree(cfHandle); taosMemoryFree(pCompare); taosMemoryFree(params); taosMemoryFree(cfOpts); - return 0; } int streamStateOpenBackend(void* backend, SStreamState* pState) { @@ -804,12 +773,13 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { pState->pTdbState->cfOpts = inst->cfOpt; pState->pTdbState->dbOpt = handle->dbOpt; pState->pTdbState->param = inst->param; - pState->pTdbState->pBackendHandle = handle; + pState->pTdbState->pBackend = handle; pState->pTdbState->pComparNode = inst->pCompareNode; taosThreadMutexUnlock(&handle->cfMutex); return 0; } taosThreadMutexUnlock(&handle->cfMutex); + return 0; char* err = NULL; int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); @@ -839,15 +809,6 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { pCompare[i] = compare; } rocksdb_column_family_handle_t** cfHandle = taosMemoryMalloc(cfLen * sizeof(rocksdb_column_family_handle_t*)); - for (int i = 0; i < cfLen; i++) { - char buf[128] = {0}; - GEN_COLUMN_FAMILY_NAME(buf, pState->pTdbState->idstr, ginitDict[i].key); - cfHandle[i] = rocksdb_create_column_family(handle->db, cfOpt[i], buf, &err); - if (err != NULL) { - qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); - taosMemoryFreeClear(err); - } - } pState->pTdbState->rocksdb = handle->db; pState->pTdbState->pHandle = cfHandle; pState->pTdbState->writeOpts = rocksdb_writeoptions_create(); @@ -855,8 +816,9 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { pState->pTdbState->cfOpts = (rocksdb_options_t**)cfOpt; pState->pTdbState->dbOpt = handle->dbOpt; pState->pTdbState->param = param; - pState->pTdbState->pBackendHandle = handle; + pState->pTdbState->pBackend = handle; + taosThreadRwlockInit(&pState->pTdbState->rwLock, NULL); SCfComparator compare = {.comp = pCompare, .numOfComp = cfLen}; pState->pTdbState->pComparNode = streamBackendAddCompare(handle, &compare); // rocksdb_writeoptions_disable_WAL(pState->pTdbState->writeOpts, 1); @@ -865,7 +827,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { } void streamStateCloseBackend(SStreamState* pState, bool remove) { - SBackendHandle* pHandle = pState->pTdbState->pBackendHandle; + SBackendHandle* pHandle = pState->pTdbState->pBackend; taosThreadMutexLock(&pHandle->cfMutex); RocksdbCfInst** ppInst = taosHashGet(pHandle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1); if (ppInst != NULL && *ppInst != NULL) { @@ -887,7 +849,8 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { char* err = NULL; if (remove) { for (int i = 0; i < cfLen; i++) { - rocksdb_drop_column_family(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[i], &err); + if (pState->pTdbState->pHandle[i] != NULL) + rocksdb_drop_column_family(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[i], &err); if (err != NULL) { qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); taosMemoryFreeClear(err); @@ -896,7 +859,8 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { } else { rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); for (int i = 0; i < cfLen; i++) { - rocksdb_flush_cf(pState->pTdbState->rocksdb, flushOpt, pState->pTdbState->pHandle[i], &err); + if (pState->pTdbState->pHandle[i] != NULL) + rocksdb_flush_cf(pState->pTdbState->rocksdb, flushOpt, pState->pTdbState->pHandle[i], &err); if (err != NULL) { qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); taosMemoryFreeClear(err); @@ -906,7 +870,9 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { } for (int i = 0; i < cfLen; i++) { - rocksdb_column_family_handle_destroy(pState->pTdbState->pHandle[i]); + if (pState->pTdbState->pHandle[i] != NULL) { + rocksdb_column_family_handle_destroy(pState->pTdbState->pHandle[i]); + } } taosMemoryFreeClear(pState->pTdbState->pHandle); for (int i = 0; i < cfLen; i++) { @@ -915,7 +881,7 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { } if (remove) { - streamBackendDelCompare(pState->pTdbState->pBackendHandle, pState->pTdbState->pComparNode); + streamBackendDelCompare(pState->pTdbState->pBackend, pState->pTdbState->pComparNode); } rocksdb_writeoptions_destroy(pState->pTdbState->writeOpts); pState->pTdbState->writeOpts = NULL; @@ -924,6 +890,8 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { pState->pTdbState->readOpts = NULL; taosMemoryFreeClear(pState->pTdbState->cfOpts); taosMemoryFreeClear(pState->pTdbState->param); + + taosThreadRwlockDestroy(&pState->pTdbState->rwLock); pState->pTdbState->rocksdb = NULL; } void streamStateDestroyCompar(void* arg) { @@ -934,14 +902,38 @@ void streamStateDestroyCompar(void* arg) { taosMemoryFree(comp->comp); } -int streamGetInit(const char* funcName) { +int streamGetInit(SStreamState* pState, const char* funcName) { + int idx = -1; size_t len = strlen(funcName); for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { if (len == ginitDict[i].len && strncmp(funcName, ginitDict[i].key, strlen(funcName)) == 0) { - return i; + idx = i; + break; } } - return -1; + if (idx != -1) { + rocksdb_column_family_handle_t* cf = NULL; + taosThreadRwlockRdlock(&pState->pTdbState->rwLock); + cf = pState->pTdbState->pHandle[idx]; + taosThreadRwlockUnlock(&pState->pTdbState->rwLock); + if (cf == NULL) { + char buf[128] = {0}; + GEN_COLUMN_FAMILY_NAME(buf, pState->pTdbState->idstr, ginitDict[idx].key); + char* err = NULL; + cf = rocksdb_create_column_family(pState->pTdbState->rocksdb, pState->pTdbState->cfOpts[idx], buf, &err); + if (err != NULL) { + idx = -1; + qError("failed to to open cf, %p 0x%" PRIx64 "-%d_%s, reason:%s", pState, pState->streamId, pState->taskId, + funcName, err); + taosMemoryFree(err); + } + taosThreadRwlockWrlock(&pState->pTdbState->rwLock); + pState->pTdbState->pHandle[idx] = cf; + taosThreadRwlockUnlock(&pState->pTdbState->rwLock); + } + } + + return idx; } bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len) { rocksdb_iter_seek(iter, buf, len); @@ -955,7 +947,7 @@ bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len } rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt) { - int idx = streamGetInit(cfName); + int idx = streamGetInit(pState, cfName); if (snapshot != NULL) { *snapshot = (rocksdb_snapshot_t*)rocksdb_create_snapshot(pState->pTdbState->rocksdb); @@ -974,7 +966,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa code = 0; \ char buf[128] = {0}; \ char* err = NULL; \ - int i = streamGetInit(funcname); \ + int i = streamGetInit(pState, funcname); \ if (i < 0) { \ qWarn("streamState failed to get cf name: %s", funcname); \ code = -1; \ @@ -1004,7 +996,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa code = 0; \ char buf[128] = {0}; \ char* err = NULL; \ - int i = streamGetInit(funcname); \ + int i = streamGetInit(pState, funcname); \ if (i < 0) { \ qWarn("streamState failed to get cf name: %s", funcname); \ code = -1; \ @@ -1051,7 +1043,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa code = 0; \ char buf[128] = {0}; \ char* err = NULL; \ - int i = streamGetInit(funcname); \ + int i = streamGetInit(pState, funcname); \ if (i < 0) { \ qWarn("streamState failed to get cf name: %s_%s", pState->pTdbState->idstr, funcname); \ code = -1; \ @@ -1979,7 +1971,7 @@ void streamStateClearBatch(void* pBatch) { rocksdb_writebatch_clear((rocksdb_ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rocksdb_writebatch_t*)pBatch); } int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { - int i = streamGetInit(cfName); + int i = streamGetInit(pState, cfName); if (i < 0) { qError("streamState failed to put to cf name:%s", cfName); From 57a4e56a616f8714460620c02fb55fd9fb07344e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 26 May 2023 20:30:03 +0800 Subject: [PATCH 108/238] change parameter --- source/libs/stream/src/streamBackendRocksdb.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 37dd7f6297..c9538f0b2a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -779,7 +779,6 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { return 0; } taosThreadMutexUnlock(&handle->cfMutex); - return 0; char* err = NULL; int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); From 479a1172663105eceff695010910555f79da2e89 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 26 May 2023 13:03:47 +0000 Subject: [PATCH 109/238] Merge branch 'enh/dev3.0' into enh/addCompileError --- include/libs/function/function.h | 65 ++++++++++++++++---------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index de2494ab3c..e015f4182e 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -21,8 +21,8 @@ extern "C" { #endif #include "tcommon.h" -#include "tvariant.h" #include "tsimplehash.h" +#include "tvariant.h" struct SqlFunctionCtx; struct SResultRowEntryInfo; @@ -77,7 +77,7 @@ enum { enum { MAIN_SCAN = 0x0u, REVERSE_SCAN = 0x1u, // todo remove it - PRE_SCAN = 0x2u, // pre-scan belongs to the main scan and occurs before main scan + PRE_SCAN = 0x2u, // pre-scan belongs to the main scan and occurs before main scan }; typedef struct SPoint1 { @@ -130,43 +130,44 @@ typedef struct SSerializeDataHandle { // incremental state storage typedef struct STdbState { - void* rocksdb; - void** pHandle; - void* writeOpts; - void* readOpts; - void** cfOpts; - void* dbOpt; - struct SStreamTask* pOwner; - void* param; - void* env; - SListNode* pComparNode; - void* pBackendHandle; + void *rocksdb; + void **pHandle; + void *writeOpts; + void *readOpts; + void **cfOpts; + void *dbOpt; + struct SStreamTask *pOwner; + void *param; + void *env; + SListNode *pComparNode; + void *pBackend; char idstr[64]; - void* compactFactory; + void *compactFactory; + TdThreadRwlock rwLock; - void* db; - void* pStateDb; - void* pFuncStateDb; - void* pFillStateDb; // todo refactor - void* pSessionStateDb; - void* pParNameDb; - void* pParTagDb; - void* txn; + void *db; + void *pStateDb; + void *pFuncStateDb; + void *pFillStateDb; // todo refactor + void *pSessionStateDb; + void *pParNameDb; + void *pParTagDb; + void *txn; } STdbState; typedef struct { - STdbState* pTdbState; - struct SStreamFileState* pFileState; - int32_t number; - SSHashObj* parNameMap; - int64_t checkPointId; - int32_t taskId; - int64_t streamId; + STdbState *pTdbState; + struct SStreamFileState *pFileState; + int32_t number; + SSHashObj *parNameMap; + int64_t checkPointId; + int32_t taskId; + int64_t streamId; } SStreamState; typedef struct SFunctionStateStore { - int32_t (*streamStateFuncPut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); - int32_t (*streamStateFuncGet)(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen); + int32_t (*streamStateFuncPut)(SStreamState *pState, const SWinKey *key, const void *value, int32_t vLen); + int32_t (*streamStateFuncGet)(SStreamState *pState, const SWinKey *key, void **ppVal, int32_t *pVLen); } SFunctionStateStore; // sql function runtime context @@ -180,7 +181,7 @@ typedef struct SqlFunctionCtx { int16_t functionId; // function id char *pOutput; // final result output buffer, point to sdata->data // input parameter, e.g., top(k, 20), the number of results of top query is kept in param - SFunctParam *param; + SFunctParam *param; // corresponding output buffer for timestamp of each result, e.g., diff/csum SColumnInfoData *pTsOutput; int32_t numOfParams; From 4b931d3007275100e30d2425bb8692a3b5017898 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 26 May 2023 13:59:16 +0000 Subject: [PATCH 110/238] Merge branch 'enh/dev3.0' into enh/addCompileError --- cmake/cmake.define | 2 +- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 35abe512f7..f3caf49da3 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -124,7 +124,7 @@ ELSE () SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -g3 -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -O3 -s -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -g3 -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ENDIF () # disable all assert diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 326c081a5b..437aa14324 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -807,7 +807,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { rocksdb_options_set_comparator((rocksdb_options_t*)cfOpt[i], compare); pCompare[i] = compare; } - rocksdb_column_family_handle_t** cfHandle = taosMemoryMalloc(cfLen * sizeof(rocksdb_column_family_handle_t*)); + rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(1, cfLen * sizeof(rocksdb_column_family_handle_t*)); pState->pTdbState->rocksdb = handle->db; pState->pTdbState->pHandle = (void**)cfHandle; pState->pTdbState->writeOpts = rocksdb_writeoptions_create(); From 2f5fe2dec6a8056d8b41bc89fbc1ed5ed9d8682a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 26 May 2023 14:01:05 +0000 Subject: [PATCH 111/238] Merge branch 'enh/dev3.0' into enh/addCompileError --- source/libs/stream/src/streamBackendRocksdb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 437aa14324..92ef66cd30 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1104,8 +1104,10 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { } char* err = NULL; - rocksdb_delete_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->writeOpts, pState->pTdbState->pHandle[1], - sKeyStr, sLen, eKeyStr, eLen, &err); + if (pState->pTdbState->pHandle[1] != NULL) { + rocksdb_delete_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->writeOpts, pState->pTdbState->pHandle[1], + sKeyStr, sLen, eKeyStr, eLen, &err); + } // rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[0], sKeyStr, sLen, eKeyStr, // eLen); if (err != NULL) { From 6368b27499bebda87ef4059d1b542ee9bb67c2c2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 29 May 2023 08:50:57 +0800 Subject: [PATCH 112/238] fix: sliding case issue --- tests/script/tsim/parser/sliding.sim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/script/tsim/parser/sliding.sim b/tests/script/tsim/parser/sliding.sim index 1cb4cb5340..7aa69ce9a9 100644 --- a/tests/script/tsim/parser/sliding.sim +++ b/tests/script/tsim/parser/sliding.sim @@ -450,10 +450,11 @@ endi print ====================>check boundary check crash at client side sql select count(*) from sliding_mt0 where ts>now and ts < now-1h; +sql select sum(c1) from sliding_tb0 interval(1a) sliding(1a); + print ========================query on super table print ========================error case -sql_error select sum(c1) from sliding_tb0 interval(1a) sliding(1a); sql_error select sum(c1) from sliding_tb0 interval(10a) sliding(12a); sql_error select sum(c1) from sliding_tb0 sliding(1n) interval(1y); sql_error select sum(c1) from sliding_tb0 interval(-1y) sliding(1n); From de739ce086d1388a7adfc18034760584673ed425 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 May 2023 01:11:13 +0000 Subject: [PATCH 113/238] Merge branch 'enh/dev3.0' into enh/addCompileError --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 92ef66cd30..b1a6eba42f 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -910,7 +910,7 @@ int streamGetInit(SStreamState* pState, const char* funcName) { break; } } - if (idx != -1) { + if (pState != NULL && idx != -1) { rocksdb_column_family_handle_t* cf = NULL; taosThreadRwlockRdlock(&pState->pTdbState->rwLock); cf = pState->pTdbState->pHandle[idx]; From 0be1858b4be796550bbd05ef7054a90b4edcacc1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 May 2023 01:50:43 +0000 Subject: [PATCH 114/238] Merge branch 'enh/dev3.0' into enh/addCompileError --- source/libs/stream/src/streamBackendRocksdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index b1a6eba42f..793575f59a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -749,6 +749,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t RocksdbCfInst* inst = *pIter; SCfComparator compare = {.comp = inst->pCompares, .numOfComp = cfLen}; inst->pCompareNode = streamBackendAddCompare(handle, &compare); + pIter = taosHashIterate(handle->cfInst, pIter); } taosMemoryFree(cfHandle); From 953a69a829a4487946cd23fe701a33b0d0bef16c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 May 2023 10:12:27 +0800 Subject: [PATCH 115/238] refactor: do some internal refactor. --- include/libs/executor/storageapi.h | 146 ++++------------------ source/dnode/vnode/inc/vnode.h | 108 +--------------- source/dnode/vnode/src/inc/vnodeInt.h | 14 +++ source/dnode/vnode/src/meta/metaCache.c | 6 +- source/dnode/vnode/src/meta/metaSma.c | 2 + source/dnode/vnode/src/tsdb/tsdbRead.c | 17 +-- source/dnode/vnode/src/vnd/vnodeInitApi.c | 6 +- source/libs/executor/src/executil.c | 7 +- 8 files changed, 69 insertions(+), 237 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 9ed47c94f8..b89293f8f0 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -103,63 +103,6 @@ typedef struct SRowBuffPos { bool beUsed; } SRowBuffPos; -// int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); -// int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, -// SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); -// int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, -// SArray *pTableUids); -// void *tsdbCacherowsReaderClose(void *pReader); -// int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); - -// int32_t tsdbReaderOpen(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, -// SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); -// int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); -// void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); -// void tsdbReaderClose(STsdbReader *pReader); -// int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); -// int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); -// void tsdbReleaseDataBlock(STsdbReader *pReader); -// SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); -// int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); -// int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); -// int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); -// void *tsdbGetIdx(void *pMeta); -// void *tsdbGetIvtIdx(void *pMeta); -// uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); -// void tsdbReaderSetCloseFlag(STsdbReader *pReader); -// int64_t tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr); - -// int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); -// int32_t vnodeGetCtbIdListByFilter(void *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); -// int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); -// void *vnodeGetIdx(void *pVnode); -// void *vnodeGetIvtIdx(void *pVnode); - -// void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); -// void metaReaderReleaseLock(SMetaReader *pReader); -// void metaReaderClear(SMetaReader *pReader); -// int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -// int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); -// int metaGetTableEntryByName(SMetaReader *pReader, const char *name); -// int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); -// int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); -// const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); -// int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); -// -// int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); -// int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); -// int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); -// bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); -// int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, -// bool *acquired); -// int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, -// int32_t payloadLen, double selectivityRatio); -// int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); -// tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); -// int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); -// int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, -// int32_t payloadLen); - // tq typedef struct SMetaTableInfo { int64_t suid; @@ -202,29 +145,10 @@ typedef struct { // int32_t destroySnapContext(SSnapContext *ctx); /*-------------------------------------------------new api format---------------------------------------------------*/ - -// typedef int32_t (*__store_reader_(STsdbReader *pReader, const void *pTableList, int32_t num); -// typedef void (*tsdbReaderSetId(STsdbReader *pReader, const char *idstr); -// typedef void (*tsdbReaderClose(STsdbReader *pReader); -// typedef int32_t (*tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); -// typedef int32_t (*tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); -// typedef void (*tsdbReleaseDataBlock(STsdbReader *pReader); -// typedef SSDataBlock * (*tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); -// typedef int32_t (*tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); -// typedef int32_t (*tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); -// typedef int64_t (*tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); -// typedef void * (*tsdbGetIdx(void *pMeta); -// typedef void * (*tsdbGetIvtIdx(void *pMeta); -// typedef uint64_t (*tsdbGetReaderMaxVersion(STsdbReader *pReader); -// typedef void (*tsdbReaderSetCloseFlag(STsdbReader *pReader); -// typedef int64_t (*tsdbGetLastTimestamp(void* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr); - -typedef int32_t (*__store_reader_open_fn_t)(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, - int32_t numOfTables, SSDataBlock *pResBlock, void **ppReader, - const char *idstr, bool countOnly, SHashObj **pIgnoreTables); - typedef struct TsdReader { - __store_reader_open_fn_t tsdReaderOpen; + int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, + SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, + SHashObj** pIgnoreTables); void (*tsdReaderClose)(); void (*tsdSetReaderTaskId)(void *pReader, const char *pId); int32_t (*tsdSetQueryTableList)(); @@ -241,22 +165,13 @@ typedef struct TsdReader { void (*tsdReaderNotifyClosing)(); } TsdReader; - -/** - * int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); -int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, - SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); -int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, - SArray *pTableUids); -void *tsdbCacherowsReaderClose(void *pReader); - */ typedef struct SStoreCacheReader { - int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, - SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); - void *(*closeReader)(void *pReader); - int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, - SArray *pTableUidList); - int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); + int32_t (*openReader)(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, + SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); + void *(*closeReader)(void *pReader); + int32_t (*retrieveRows)(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, + SArray *pTableUidList); + int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); } SStoreCacheReader; /*------------------------------------------------------------------------------------------------------------------*/ @@ -309,16 +224,10 @@ typedef struct SStoreTqReader { } SStoreTqReader; typedef struct SStoreSnapshotFn { - /* - int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); - SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); - int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); - int32_t destroySnapContext(SSnapContext *ctx); - */ - int32_t (*createSnapshot)(); - int32_t (*destroySnapshot)(); - SMetaTableInfo (*getMetaTableInfoFromSnapshot)(); - int32_t (*getTableInfoFromSnapshot)(); + int32_t (*createSnapshot)(SSnapContext *ctx, int64_t uid); + int32_t (*destroySnapshot)(SSnapContext *ctx); + SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx); + int32_t (*getTableInfoFromSnapshot)(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid); } SStoreSnapshotFn; /** @@ -327,7 +236,6 @@ void metaReaderReleaseLock(SMetaReader *pReader); void metaReaderClear(SMetaReader *pReader); int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); -int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); @@ -340,7 +248,6 @@ int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *k bool *acquired); int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen, double selectivityRatio); -int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, @@ -350,29 +257,28 @@ int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, in typedef struct SStoreMeta { SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor void (*closeTableMetaCursor)(SMTbCursor *pTbCur); // metaCloseTbCursor - int32_t (*cursorNext)(); // metaTbCursorNext - int32_t (*cursorPrev)(); // metaTbCursorPrev + int32_t (*cursorNext)(SMTbCursor *pTbCur, ETableType jumpTableType); // metaTbCursorNext + int32_t (*cursorPrev)(SMTbCursor *pTbCur, ETableType jumpTableType); // metaTbCursorPrev - int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); - int32_t (*getTableTagsByUid)(void *pVnode, int64_t suid, SArray *uidList); + int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); + int32_t (*getTableTagsByUid)(void *pVnode, int64_t suid, SArray *uidList); const void *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it - int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); - int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType); - int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName); - bool (*isTableExisted)(void *pVnode, tb_uid_t uid); + int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); + int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType); + int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName); + bool (*isTableExisted)(void *pVnode, tb_uid_t uid); /** - * int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, - int32_t payloadLen, double selectivityRatio); -int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); -tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); */ - void (*getCachedTableList)(); - void (*putTableListIntoCache)(); + int32_t (*getCachedTableList)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, + bool* acquireRes); + int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen, double selectivityRatio); +// int32_t (*clearCachedTableList)(void* pVnode, uint64_t suid); /** * diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 72c8dce52f..af92291456 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -100,18 +100,11 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit); // meta -typedef struct SMeta SMeta; // todo: remove -typedef struct SMetaReader SMetaReader; -typedef struct SMetaEntry SMetaEntry; - -#define META_READER_NOLOCK 0x1 - void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI); void metaReaderReleaseLock(SMetaReader *pReader); void metaReaderClear(SMetaReader *pReader); int32_t metaReaderGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaReaderGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); -int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *uidList); int32_t metaGetTableTagsByUids(void* pVnode, int64_t suid, SArray *uidList); int32_t metaReadNext(SMetaReader *pReader); @@ -122,11 +115,10 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); bool metaIsTableExist(void* pVnode, tb_uid_t uid); -int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, +int32_t metaGetCachedTableUidList(void *pVnode, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, bool *acquired); -int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, +int32_t metaUidFilterCachePut(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen, double selectivityRatio); -int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); int32_t metaGetCachedTbGroup(SMeta *pMeta, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); @@ -135,25 +127,9 @@ int32_t metaPutTbGroupToCache(SMeta *pMeta, uint64_t suid, const void *pKey, in int64_t metaGetTbNum(SMeta *pMeta); int64_t metaGetNtbNum(SMeta *pMeta); -//typedef struct { -// int64_t uid; -// int64_t ctbNum; -//} SMetaStbStats; + int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); -#if 1 // refact APIs below (TODO) -typedef SVCreateTbReq STbCfg; -typedef SVCreateTSmaReq SSmaCfg; - -typedef struct SMTbCursor SMTbCursor; - -SMTbCursor *metaOpenTbCursor(void *pVnode); -void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); - -#endif - // tsdb typedef struct STsdbReader STsdbReader; @@ -168,8 +144,8 @@ typedef struct STsdbReader STsdbReader; #define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST 0x8 -int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, +int32_t tsdbReaderOpen(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, + SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly, SHashObj **pIgnoreTables); int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); @@ -201,32 +177,11 @@ size_t tsdbCacheGetUsage(SVnode *pVnode); int32_t tsdbCacheGetElems(SVnode *pVnode); //// tq -//typedef struct SMetaTableInfo { -// int64_t suid; -// int64_t uid; -// SSchemaWrapper *schema; -// char tbName[TSDB_TABLE_NAME_LEN]; -//} SMetaTableInfo; - typedef struct SIdInfo { int64_t version; int32_t index; } SIdInfo; -//typedef struct SSnapContext { -// SMeta *pMeta; -// int64_t snapVersion; -// TBC *pCur; -// int64_t suid; -// int8_t subType; -// SHashObj *idVersion; -// SHashObj *suidInfo; -// SArray *idList; -// int32_t index; -// bool withMeta; -// bool queryMeta; // true-get meta, false-get data -//} SSnapContext; - typedef struct STqReader { SPackedData msg; SSubmitReq2 submit; @@ -345,59 +300,6 @@ struct SVnodeCfg { #define TABLE_ROLLUP_ON ((int8_t)0x1) #define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) #define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) -//struct SMetaEntry { -// int64_t version; -// int8_t type; -// int8_t flags; // TODO: need refactor? -// tb_uid_t uid; -// char *name; -// union { -// struct { -// SSchemaWrapper schemaRow; -// SSchemaWrapper schemaTag; -// SRSmaParam rsmaParam; -// } stbEntry; -// struct { -// int64_t ctime; -// int32_t ttlDays; -// int32_t commentLen; -// char *comment; -// tb_uid_t suid; -// uint8_t *pTags; -// } ctbEntry; -// struct { -// int64_t ctime; -// int32_t ttlDays; -// int32_t commentLen; -// char *comment; -// int32_t ncid; // next column id -// SSchemaWrapper schemaRow; -// } ntbEntry; -// struct { -// STSma *tsma; -// } smaEntry; -// }; -// -// uint8_t *pBuf; -//}; - -//struct SMetaReader { -// int32_t flags; -// SMeta *pMeta; -// SDecoder coder; -// SMetaEntry me; -// void *pBuf; -// int32_t szBuf; -//}; - -//struct SMTbCursor { -// TBC *pDbc; -// void *pKey; -// void *pVal; -// int32_t kLen; -// int32_t vLen; -// SMetaReader mr; -//}; #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 3f79fe5fd1..7ae05b9554 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -103,6 +103,17 @@ struct SQueryNode { _query_reseek_func_t reseek; }; +#if 1 // refact APIs below (TODO) +typedef SVCreateTbReq STbCfg; +typedef SVCreateTSmaReq SSmaCfg; + +SMTbCursor *metaOpenTbCursor(void *pVnode); +void metaCloseTbCursor(SMTbCursor *pTbCur); +int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); +int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); + +#endif + void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); void* vnodeBufPoolMallocAligned(SVBufPool* pPool, int size); void vnodeBufPoolFree(SVBufPool* pPool, void* p); @@ -143,6 +154,8 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); +int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); + int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); @@ -477,6 +490,7 @@ struct SCompactInfo { void initStorageAPI(SStorageAPI* pAPI); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 436ca1abd3..054b9f6f24 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -499,8 +499,9 @@ static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, ASSERT(keyLen == sizeof(uint64_t) * 2); } -int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, +int32_t metaGetCachedTableUidList(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; int32_t vgId = TD_VID(pMeta->pVnode); // generate the composed key for LRU cache @@ -603,9 +604,10 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL } // check both the payload size and selectivity ratio -int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, +int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio) { int32_t code = 0; + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; int32_t vgId = TD_VID(pMeta->pVnode); if (selectivityRatio > tsSelectivityRatio) { diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 3fb491848a..a49848f442 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -13,8 +13,10 @@ * along with this program. If not, see . */ +#include "vnodeInt.h" #include "meta.h" + static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 268d17bacb..362934ec84 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -754,7 +754,7 @@ static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacit return terrno; } -static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity, +static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity, SSDataBlock* pResBlock, const char* idstr) { int32_t code = 0; int8_t level = 0; @@ -4396,11 +4396,12 @@ static void freeSchemaFunc(void* param) { } // ====================================== EXPOSED APIs ====================================== -int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr, bool countOnly, SHashObj** pIgnoreTables) { +int32_t tsdbReaderOpen(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, + SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, SHashObj** pIgnoreTables) { STimeWindow window = pCond->twindows; + SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); - int32_t capacity = pVnode->config.tsdbCfg.maxRows; + int32_t capacity = pConf->tsdbCfg.maxRows; if (pResBlock != NULL) { blockDataEnsureCapacity(pResBlock, capacity); } @@ -4431,7 +4432,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } // here we only need one more row, so the capacity is set to be ONE. - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, pResBlock, idstr); + code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[0], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -4445,7 +4446,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } pCond->order = order; - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, pResBlock, idstr); + code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[1], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -4495,7 +4496,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL goto _err; } - pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); + pReader->status.pLDataIter = taosMemoryCalloc(pConf->sttTrigger, sizeof(SLDataIter)); if (pReader->status.pLDataIter == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -5546,7 +5547,7 @@ int64_t tsdbGetLastTimestamp(SVnode* pVnode, void* pTableList, int32_t numOfTabl int64_t key = INT64_MIN; for(int32_t i = 0; i < numOfTables; ++i) { - int32_t code = tsdbReaderOpen(pVnode, &cond, &pTableKeyInfo[i], 1, pBlock, &pReader, pIdStr, false, NULL); + int32_t code = tsdbReaderOpen(pVnode, &cond, &pTableKeyInfo[i], 1, pBlock, (void**)&pReader, pIdStr, false, NULL); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 1cc985f7df..7ca93df39e 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -41,7 +41,8 @@ void initStorageAPI(SStorageAPI* pAPI) { } void initTsdbReaderAPI(TsdReader* pReader) { - pReader->tsdReaderOpen = (__store_reader_open_fn_t)tsdbReaderOpen; + pReader->tsdReaderOpen = (int32_t(*)(void*, SQueryTableDataCond*, void*, int32_t, SSDataBlock*, void**, const char*, + bool, SHashObj**))tsdbReaderOpen; pReader->tsdReaderClose = tsdbReaderClose; pReader->tsdNextDataBlock = tsdbNextDataBlock; @@ -87,6 +88,9 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor pMeta->storeGetTableList = vnodeGetTableList; + + pMeta->getCachedTableList = metaGetCachedTableUidList; + pMeta->putCachedTableList = metaUidFilterCachePut; } void initTqAPI(SStoreTqReader* pTq) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 562a6daf8a..f51527b944 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -494,8 +494,8 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf listNode->pNodeList = group; genTbGroupDigest((SNode*)listNode, digest, &context); nodesFree(listNode); - - pAPI->metaFn.getCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); + ASSERT(0); +// pAPI->metaFn.getCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList, true); if (tableList) { taosArrayDestroy(pTableListInfo->pTableList); pTableListInfo->pTableList = tableList; @@ -630,9 +630,10 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf info->groupId = calcGroupId(keyBuf, len); } + ASSERT(0); if (tsTagFilterCache) { tableList = taosArrayDup(pTableListInfo->pTableList, NULL); - pAPI->metaFn.putTableListIntoCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); +// pAPI->metaFn.putCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); } // int64_t st2 = taosGetTimestampUs(); From 342ce14c7daf079ed1a4a2bf46e074ba3452f84d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 May 2023 02:37:47 +0000 Subject: [PATCH 116/238] Merge branch 'enh/3.0' into fix/shared_link_rocksdb --- cmake/cmake.define | 4 ++-- contrib/CMakeLists.txt | 5 ++++- source/dnode/vnode/CMakeLists.txt | 23 +++++++++++++++++++++++ source/libs/stream/CMakeLists.txt | 10 ++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 0d5c21604a..2e55b954b7 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -123,8 +123,8 @@ ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ELSE () - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -g3 -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -g3-Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ENDIF () # disable all assert diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ec48a85c33..3c0c50b73b 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -268,8 +268,11 @@ if(${BUILD_WITH_ROCKSDB}) option(WITH_BENCHMARK_TOOLS "" OFF) option(WITH_TOOLS "" OFF) option(WITH_LIBURING "" OFF) - + IF (TD_LINUX) option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON) + ELSE() + option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) + ENDIF() add_subdirectory(rocksdb EXCLUDE_FROM_ALL) target_include_directories( rocksdb diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 5c752ba265..b18cb8e282 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -87,6 +87,7 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar" PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include" ) +IF (TD_LINUX) target_link_libraries( vnode PUBLIC os @@ -107,6 +108,28 @@ target_link_libraries( PUBLIC stream PUBLIC index ) +ELSE() +target_link_libraries( + vnode + PUBLIC os + PUBLIC util + PUBLIC common + PUBLIC tfs + PUBLIC wal + PUBLIC qworker + PUBLIC sync + PUBLIC executor + PUBLIC scheduler + PUBLIC tdb + + # PUBLIC bdb + # PUBLIC scalar + PUBLIC rocksdb + PUBLIC transport + PUBLIC stream + PUBLIC index +) +ENDIF() IF (TD_GRANT) TARGET_LINK_LIBRARIES(vnode PUBLIC grant) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 72517a88da..fa6c709c8f 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -6,12 +6,22 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + if(${BUILD_WITH_ROCKSDB}) + IF (TD_LINUX) target_link_libraries( stream PUBLIC rocksdb-shared tdb PRIVATE os util transport qcom executor wal index ) + ELSE() + target_link_libraries( + stream + PUBLIC rocksdb tdb + PRIVATE os util transport qcom executor wal index + ) + + ENDIF() target_include_directories( stream From 43c0656df2fa0ff5f864185eed902cb8a2fa1d52 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 29 May 2023 10:45:26 +0800 Subject: [PATCH 117/238] fix: if block row num is less than definit min rows, set the bucket index to 0 --- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5d970ce6c3..371ea77b4e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -5242,6 +5242,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { } static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) { + if (numOfRows < startRow) { + return 0; + } int32_t bucketIndex = ((numOfRows - startRow) / bucketRange); if (bucketIndex == numOfBucket) { bucketIndex -= 1; From d0b9abcc7bf59f282e7b35b5b27306c8fa0cf5d5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 May 2023 10:46:53 +0800 Subject: [PATCH 118/238] merge 3.0 --- cmake/cmake.define | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 2e55b954b7..f3caf49da3 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -124,7 +124,7 @@ ELSE () SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -g3 -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -g3-Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -g3 -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ENDIF () # disable all assert From 8f61e2d6efc80e5ff0a06c06916b8beb5e93a565 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 May 2023 10:50:50 +0800 Subject: [PATCH 119/238] refactor: do some internal refactor. --- include/libs/executor/storageapi.h | 65 ++++++++++------------- source/dnode/vnode/inc/vnode.h | 8 ++- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaCache.c | 6 ++- source/dnode/vnode/src/meta/metaQuery.c | 2 - source/dnode/vnode/src/vnd/vnodeInitApi.c | 5 +- source/libs/executor/src/executil.c | 7 ++- source/libs/executor/src/scanoperator.c | 2 +- 8 files changed, 45 insertions(+), 51 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index b89293f8f0..4cfbf2c2fa 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -144,25 +144,26 @@ typedef struct { // int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); // int32_t destroySnapContext(SSnapContext *ctx); +// clang-format off /*-------------------------------------------------new api format---------------------------------------------------*/ typedef struct TsdReader { - int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, + int32_t (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly, SHashObj** pIgnoreTables); - void (*tsdReaderClose)(); - void (*tsdSetReaderTaskId)(void *pReader, const char *pId); - int32_t (*tsdSetQueryTableList)(); - int32_t (*tsdNextDataBlock)(); + void (*tsdReaderClose)(); + void (*tsdSetReaderTaskId)(void *pReader, const char *pId); + int32_t (*tsdSetQueryTableList)(); + int32_t (*tsdNextDataBlock)(); - int32_t (*tsdReaderRetrieveBlockSMAInfo)(); + int32_t (*tsdReaderRetrieveBlockSMAInfo)(); SSDataBlock *(*tsdReaderRetrieveDataBlock)(); - void (*tsdReaderReleaseDataBlock)(); + void (*tsdReaderReleaseDataBlock)(); - int32_t (*tsdReaderResetStatus)(); - int32_t (*tsdReaderGetDataBlockDistInfo)(); - int64_t (*tsdReaderGetNumOfInMemRows)(); - void (*tsdReaderNotifyClosing)(); + int32_t (*tsdReaderResetStatus)(); + int32_t (*tsdReaderGetDataBlockDistInfo)(); + int64_t (*tsdReaderGetNumOfInMemRows)(); + void (*tsdReaderNotifyClosing)(); } TsdReader; typedef struct SStoreCacheReader { @@ -174,6 +175,8 @@ typedef struct SStoreCacheReader { int32_t (*reuseReader)(void *pReader, void *pTableIdList, int32_t numOfTables); } SStoreCacheReader; +// clang-format on + /*------------------------------------------------------------------------------------------------------------------*/ /* void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); @@ -240,7 +243,6 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); -int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); @@ -250,13 +252,12 @@ int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, in int32_t payloadLen, double selectivityRatio); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); -int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, - int32_t payloadLen); +int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); */ typedef struct SStoreMeta { - SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor - void (*closeTableMetaCursor)(SMTbCursor *pTbCur); // metaCloseTbCursor + SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor + void (*closeTableMetaCursor)(SMTbCursor *pTbCur); // metaCloseTbCursor int32_t (*cursorNext)(SMTbCursor *pTbCur, ETableType jumpTableType); // metaTbCursorNext int32_t (*cursorPrev)(SMTbCursor *pTbCur, ETableType jumpTableType); // metaTbCursorPrev @@ -269,28 +270,20 @@ typedef struct SStoreMeta { int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName); bool (*isTableExisted)(void *pVnode, tb_uid_t uid); - /** -int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); -int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, - int32_t payloadLen); - */ - int32_t (*getCachedTableList)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, - bool* acquireRes); - int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, - int32_t payloadLen, double selectivityRatio); -// int32_t (*clearCachedTableList)(void* pVnode, uint64_t suid); + int32_t (*metaGetCachedTbGroup)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); + int32_t (*metaPutTbGroupToCache)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); - /** - * - */ - void *(*storeGetIndexInfo)(); - void *(*getInvertIndex)(void* pVnode); - int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] - int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList - void *storeGetVersionRange; - void *storeGetLastTimestamp; + int32_t (*getCachedTableList)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes); + int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio); - int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema + void *(*storeGetIndexInfo)(); + void *(*getInvertIndex)(void* pVnode); + int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] + int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList + void *storeGetVersionRange; + void *storeGetLastTimestamp; + + int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema // db name, vgId, numOfTables, numOfSTables int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index af92291456..fbe2f4809c 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -120,15 +120,13 @@ int32_t metaGetCachedTableUidList(void *pVnode, tb_uid_t suid, const uint8_t *k int32_t metaUidFilterCachePut(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen, double selectivityRatio); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); -int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); -int32_t metaGetCachedTbGroup(SMeta *pMeta, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); -int32_t metaPutTbGroupToCache(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, +int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); +int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); int64_t metaGetTbNum(SMeta *pMeta); -int64_t metaGetNtbNum(SMeta *pMeta); -int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); // tsdb typedef struct STsdbReader STsdbReader; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7ae05b9554..db285dc124 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -155,6 +155,7 @@ int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); +int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 054b9f6f24..8749b3ac94 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -704,7 +704,8 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { return TSDB_CODE_SUCCESS; } -int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList) { +int32_t metaGetCachedTbGroup(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; int32_t vgId = TD_VID(pMeta->pVnode); // generate the composed key for LRU cache @@ -788,9 +789,10 @@ static void freeTbGroupCachePayload(const void* key, size_t keyLen, void* value) } -int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, +int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen) { int32_t code = 0; + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; int32_t vgId = TD_VID(pMeta->pVnode); if (payloadLen > tsTagFilterResCacheSize) { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 26182baf7b..726ed6a4c8 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -700,8 +700,6 @@ int64_t metaGetTimeSeriesNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries; } -int64_t metaGetNtbNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfNTables; } - typedef struct { SMeta *pMeta; TBC *pCur; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 7ca93df39e..526b9b4e2d 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -31,10 +31,10 @@ static void initSnapshotFn(SStoreSnapshotFn* pSnapshot); void initStorageAPI(SStorageAPI* pAPI) { initTsdbReaderAPI(&pAPI->tsdReader); initMetadataAPI(&pAPI->metaFn); - initTqAPI(&pAPI->tqReaderFn); initStateStoreAPI(&pAPI->stateStore); initMetaReaderAPI(&pAPI->metaReaderFn); initMetaFilterAPI(&pAPI->metaFilter); + initTqAPI(&pAPI->tqReaderFn); initFunctionStateStore(&pAPI->functionStore); initCacheFn(&pAPI->cacheFn); initSnapshotFn(&pAPI->snapshotFn); @@ -91,6 +91,9 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getCachedTableList = metaGetCachedTableUidList; pMeta->putCachedTableList = metaUidFilterCachePut; + + pMeta->metaGetCachedTbGroup = metaGetCachedTbGroup; + pMeta->metaPutTbGroupToCache = metaPutTbGroupToCache; } void initTqAPI(SStoreTqReader* pTq) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f51527b944..53b100da22 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -494,8 +494,8 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf listNode->pNodeList = group; genTbGroupDigest((SNode*)listNode, digest, &context); nodesFree(listNode); - ASSERT(0); -// pAPI->metaFn.getCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList, true); + + pAPI->metaFn.metaGetCachedTbGroup(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); if (tableList) { taosArrayDestroy(pTableListInfo->pTableList); pTableListInfo->pTableList = tableList; @@ -630,10 +630,9 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf info->groupId = calcGroupId(keyBuf, len); } - ASSERT(0); if (tsTagFilterCache) { tableList = taosArrayDup(pTableListInfo->pTableList, NULL); -// pAPI->metaFn.putCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); + pAPI->metaFn.metaPutTbGroupToCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); } // int64_t st2 = taosGetTimestampUs(); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3984e9a792..2d2f377041 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3446,7 +3446,7 @@ static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, S uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; - int64_t numOfTables = 0;//metaGetNtbNum(pInfo->readHandle.vnode); + int64_t numOfTables = 0; pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, NULL, &numOfTables); if (numOfTables != 0) { From a71878d07fb94e184991259a3fd18a5745e4afd9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 11:46:59 +0800 Subject: [PATCH 120/238] feat: support create topic as stable with conditions --- include/common/tmsg.h | 2 +- include/libs/executor/executor.h | 2 +- source/dnode/mnode/impl/src/mndScheduler.c | 20 +++++++++++++++++--- source/dnode/mnode/impl/src/mndStb.c | 4 ++-- source/dnode/mnode/impl/src/mndTopic.c | 10 ++++++++-- source/dnode/vnode/src/tq/tq.c | 15 +++++++++------ source/dnode/vnode/src/tq/tqMeta.c | 10 ++++++---- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/executor/src/executil.c | 13 +++++++------ 9 files changed, 52 insertions(+), 26 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a7de03fbce..db46858cec 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2772,8 +2772,8 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { buf = taosDecodeString(buf, &pReq->qmsg); } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { - buf = taosDecodeString(buf, &pReq->qmsg); buf = taosDecodeFixedI64(buf, &pReq->suid); + buf = taosDecodeString(buf, &pReq->qmsg); } return (void*)buf; } diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index e3a75ecabc..d64b3dd193 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -82,7 +82,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); -int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* pTagCond, void* pTagIndexCond, SArray **tableList); +int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* node, SArray **tableList); /** * set the task Id, usually used by message queue process diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 0248a195db..39c771d111 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -538,7 +538,23 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE && pTopic->ast != NULL){ + SNode *pAst = NULL; + if (nodesStringToNode(pTopic->ast, &pAst) != 0) { + mError("topic:%s, failed to create since %s", pTopic->name, terrstr()); + return -1; + } + SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; + if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { + mError("failed to create topic:%s since %s", pTopic->name, terrstr()); + nodesDestroyNode(pAst); + return -1; + } + nodesDestroyNode(pAst); + } + + if(pPlan){ int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); if (levelNum != 1) { qDestroyQueryPlan(pPlan); @@ -579,7 +595,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib mDebug("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId); - if (pTopic->subType == TOPIC_SUB_TYPE__COLUMN) { + if (pSubplan) { int32_t msgLen; pSubplan->execNode.epSet = pVgEp->epSet; @@ -591,8 +607,6 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - } else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ - pVgEp->qmsg = taosStrdup(pTopic->ast); } else { pVgEp->qmsg = taosStrdup(""); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 8b708c3e0f..2b54eb6389 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1205,7 +1205,7 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, mInfo("topic:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, subType:%d sql:%s", pTopic->name, stbFullName, suid, colId, pTopic->subType, pTopic->sql); - if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { + if (pTopic->ast == NULL) { sdbRelease(pSdb, pTopic); continue; } @@ -2247,7 +2247,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, } } - if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { + if (pTopic->ast == NULL) { sdbRelease(pSdb, pTopic); continue; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 7d71aae3f4..863055922a 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -420,6 +420,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * mError("failed to create topic:%s since %s", pCreate->name, terrstr()); taosMemoryFree(topicObj.ast); taosMemoryFree(topicObj.sql); + nodesDestroyNode(pAst); return -1; } @@ -427,6 +428,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (topicObj.ntbColIds == NULL) { taosMemoryFree(topicObj.ast); taosMemoryFree(topicObj.sql); + nodesDestroyNode(pAst); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -442,6 +444,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); taosMemoryFree(topicObj.ast); taosMemoryFree(topicObj.sql); + nodesDestroyNode(pAst); return -1; } @@ -462,8 +465,11 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.stbUid = pStb->uid; mndReleaseStb(pMnode, pStb); - topicObj.ast = taosStrdup(pCreate->ast); - topicObj.astLen = strlen(pCreate->ast) + 1; + if(pCreate->ast != NULL){ + qDebugL("topic:%s ast %s", topicObj.name, pCreate->ast); + topicObj.ast = taosStrdup(pCreate->ast); + topicObj.astLen = strlen(pCreate->ast) + 1; + } } /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/ /*topicObj.ast = NULL;*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index be3d6bc614..30c52d9fc2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -518,19 +518,22 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->execHandle.execTb.qmsg = req.qmsg; req.qmsg = NULL; - if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { - tqError("nodesStringToNode error in sub stable, since %s", terrstr()); - return -1; + if(strcmp(pHandle->execHandle.execTb.qmsg, "") != 0) { + if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s, vgId:%d, subkey:%s consumer:0x%" PRIx64, terrstr(), + pVnode->config.vgId, req.subKey, pHandle->consumerId); + return -1; + } } SArray* tbUidList = NULL; - ret = qGetTableList(req.suid, pVnode->pMeta, pVnode, pHandle->execHandle.execTb.node, NULL, &tbUidList); + ret = qGetTableList(req.suid, pVnode->pMeta, pVnode, pHandle->execHandle.execTb.node, &tbUidList); if(ret != TDB_CODE_SUCCESS) { - tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, req.subKey, pHandle->consumerId); + tqError("qGetTableList error:%d vgId:%d, subkey:%s consumer:0x%" PRIx64, ret, pVnode->config.vgId, req.subKey, pHandle->consumerId); taosArrayDestroy(tbUidList); goto end; } - tqDebug("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, req.suid); + tqDebug("tq try to get ctb for stb subscribe, vgId:%d, subkey:%s consumer:0x%" PRIx64 " suid:%" PRId64, pVnode->config.vgId, req.subKey, pHandle->consumerId, req.suid); pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index ee7af5b2bf..e93efcd3cc 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -338,13 +338,15 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - if (nodesStringToNode(handle.execHandle.execTb.qmsg, &handle.execHandle.execTb.node) != 0) { - tqError("nodesStringToNode error in sub stable, since %s", terrstr()); - return -1; + if(strcmp(handle.execHandle.execTb.qmsg, "") != 0) { + if (nodesStringToNode(handle.execHandle.execTb.qmsg, &handle.execHandle.execTb.node) != 0) { + tqError("nodesStringToNode error in sub stable, since %s", terrstr()); + return -1; + } } SArray* tbUidList = NULL; - int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, handle.execHandle.execTb.node, NULL, &tbUidList); + int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle.subKey, handle.consumerId); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 38f5307384..820a8b0eba 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1041,7 +1041,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* list = NULL; - int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, pTqHandle->execHandle.execTb.node, NULL, &list); + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); taosArrayDestroy(list); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1fb35ae271..fb889692ef 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1122,13 +1122,14 @@ _end: return code; } -int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* pTagCond, void* pTagIndexCond, SArray **tableList){ - SScanPhysiNode node = {0}; - node.suid = suid; - node.uid = suid; - node.tableType = TSDB_SUPER_TABLE; +int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* node, SArray **tableList){ + SSubplan *pSubplan = (SSubplan *)node; + SScanPhysiNode pNode = {0}; + pNode.suid = suid; + pNode.uid = suid; + pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); - int code = getTableList(metaHandle, pVnode, &node, pTagCond, pTagIndexCond, pTableListInfo, "qGetTableList"); + int code = getTableList(metaHandle, pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList"); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From f9d867f50d46cb52921e9bbc79960596af439197 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Mon, 29 May 2023 13:20:53 +0800 Subject: [PATCH 121/238] fix: TDengine build on macOS arm64 --- cmake/cmake.platform | 12 ++++++++++-- contrib/CMakeLists.txt | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index cb09bf2085..f9faf7316c 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -56,9 +56,17 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(TD_DARWIN TRUE) SET(OSTYPE "macOS") + execute_process(COMMAND geos-config --cflags OUTPUT_VARIABLE GEOS_CFLAGS) + execute_process(COMMAND geos-config --ldflags OUTPUT_VARIABLE GEOS_LDFLAGS) + string(SUBSTRING ${GEOS_CFLAGS} 2 -1 GEOS_CFLAGS) + string(REGEX REPLACE "\n" "" GEOS_CFLAGS ${GEOS_CFLAGS}) + string(SUBSTRING ${GEOS_LDFLAGS} 2 -1 GEOS_LDFLAGS) + string(REGEX REPLACE "\n" "" GEOS_LDFLAGS ${GEOS_LDFLAGS}) + MESSAGE("GEOS_CFLAGS "${GEOS_CFLAGS}) + MESSAGE("GEOS_LDFLAGS "${GEOS_LDFLAGS}) ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") - INCLUDE_DIRECTORIES(/usr/local/include) - LINK_DIRECTORIES(/usr/local/lib) + INCLUDE_DIRECTORIES(${GEOS_CFLAGS}) + LINK_DIRECTORIES(${GEOS_LDFLAGS}) IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") MESSAGE("Current system arch is arm64") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b3333dee99..eeb20ee884 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -236,6 +236,10 @@ if(${BUILD_WITH_ROCKSDB}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") endif(${TD_DARWIN}) + if (${TD_DARWIN_ARM64}) + set(HAS_ARMV8_CRC true) + endif(${TD_DARWIN_ARM64}) + if (${TD_WINDOWS}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") endif(${TD_WINDOWS}) From 60588af69476567940b6668e3c526c650a772104 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 29 May 2023 13:33:51 +0800 Subject: [PATCH 122/238] fix: add topic query --- source/libs/parser/src/parTranslater.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6de1bd4b85..b858658709 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5882,6 +5882,7 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt strcpy(realTable->table.tableAlias, pStmt->subSTbName); *pSelect = createSelectStmtImpl(true, pProjection, (SNode*)realTable); ((SSelectStmt*)*pSelect)->pWhere = nodesCloneNode(pStmt->pWhere); + pCxt->pParseCxt->topicQuery = true; code = translateQuery(pCxt, *pSelect); } From e21a2f9cce3a21ed0ea803f27cc339c7262c91ba Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Mon, 29 May 2023 14:02:36 +0800 Subject: [PATCH 123/238] check empty --- source/dnode/mnode/impl/src/mndDnode.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 73dbb243a1..c4b0e00c76 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -995,11 +995,13 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (!isonline && !force) { - terrno = TSDB_CODE_DNODE_OFFLINE; - mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), - numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); - goto _OVER; + if (numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) { + if (!isonline && !force) { + terrno = TSDB_CODE_DNODE_OFFLINE; + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); + goto _OVER; + } } code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes, force, dropReq.unsafe); From 97e0e0ea0da6e709d36b98e92017c7aef5b61ca6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 14:35:59 +0800 Subject: [PATCH 124/238] fix:add test case --- tests/system-test/7-tmq/stbFilterWhere.py | 159 ++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 tests/system-test/7-tmq/stbFilterWhere.py diff --git a/tests/system-test/7-tmq/stbFilterWhere.py b/tests/system-test/7-tmq/stbFilterWhere.py new file mode 100644 index 0000000000..baf042d71c --- /dev/null +++ b/tests/system-test/7-tmq/stbFilterWhere.py @@ -0,0 +1,159 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + tmqCom.initConsumerTable() + tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica']) + tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict["dbName"])) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + return + + def tmqCaseError(self, topicName, condition): + tdLog.printNoPrefix("======== test case error: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + tdLog.info("create topics from stb with column filter") + topicString = "create topic %s as stable %s.%s where %s" %(topicName, paraDict['dbName'], paraDict['stbName'], condition) + tdLog.info("create topic sql: %s"%topicString) + tdSql.error(topicString) + + def tmqCase(self, topicName, condition): + tdLog.printNoPrefix("======== test case: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with tag filter") + topicString = "create topic %s as stable %s.%s where %s" %(topicName, paraDict['dbName'], paraDict['stbName'], condition) + tdLog.info("create topic sql: %s"%topicString) + tdSql.execute(topicString) + + queryString = "select * from %s.%s where %s" %(paraDict['dbName'], paraDict['stbName'], condition) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicName + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + tdLog.printNoPrefix("======== test case end ...... ") + + def run(self): + tdSql.prepare() + self.prepareTestEnv() + self.tmqCaseError("t1", "c1 = 4 and t1 = 3") + self.tmqCase("t2", "2 > 1") + self.tmqCase("t3", "t4 = 'beijing'") + self.tmqCase("t4", "t4 > t3") + self.tmqCase("t5", "t3 = t4") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 3bbf5405c6f9d80521c76b065166be765d0ed435 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 29 May 2023 14:36:29 +0800 Subject: [PATCH 125/238] fix(query): invalid buf page --- source/libs/executor/inc/executil.h | 3 +++ source/util/src/tpagedbuf.c | 1 + 2 files changed, 4 insertions(+) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 5b248f9a90..0f169b0c2e 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -135,6 +135,9 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); + if (!bufPage) { + return NULL; + } if (forUpdate) { setBufPageDirty(bufPage, true); } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index fa8b5d33b7..218a47ea62 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -482,6 +482,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); if (*pInfo != *pi) { + terrno = TSDB_CODE_QRY_SYS_ERROR; uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id); return NULL; } From e36acd0a63b2396da2a277d944a441085bf92cec Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 14:45:06 +0800 Subject: [PATCH 126/238] fix:add test case --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4065ac5bee..44a08da90e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -75,6 +75,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/db.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqError.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/schema.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilterWhere.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py From 5e2d3939fef5b48fa50b9dd401a5c6a1989f48e5 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Mon, 29 May 2023 11:33:08 +0800 Subject: [PATCH 127/238] fix: page buffer cache flush wrong bytes of data into disk --- source/util/src/tpagedbuf.c | 2 +- source/util/test/CMakeLists.txt | 10 ++++- source/util/test/pageBufferTest.cpp | 65 ++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index fa8b5d33b7..acbf0bb0d3 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -166,7 +166,7 @@ static char* doFlushBufPage(SDiskbasedBuf* pBuf, SPageInfo* pg) { char* t = NULL; if ((!HAS_DATA_IN_DISK(pg)) || pg->dirty) { void* payload = GET_PAYLOAD_DATA(pg); - t = doCompressData(payload, pBuf->pageSize, &size, pBuf); + t = doCompressData(payload, pBuf->pageSize + sizeof(SFilePage), &size, pBuf); if (size < 0) { uError("failed to compress data when flushing data to disk, %s", pBuf->id); terrno = TSDB_CODE_INVALID_PARA; diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 2e307771b7..0bf06e6f44 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -75,4 +75,12 @@ target_link_libraries(rbtreeTest os util gtest_main) add_test( NAME rbtreeTest COMMAND rbtreeTest -) \ No newline at end of file +) + +# pageBufferTest +add_executable(pageBufferTest "pageBufferTest.cpp") +target_link_libraries(pageBufferTest os util gtest_main) +add_test( + NAME pageBufferTest + COMMAND pageBufferTest +) diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 00ed804930..50d3656ccd 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -157,6 +157,68 @@ void recyclePageTest() { destroyDiskbasedBuf(pBuf); } + +int saveDataToPage(SFilePage* pPg, const char* data, uint32_t len) { + memcpy(pPg->data + pPg->num, data, len); + pPg->num += len; + setBufPageDirty(pPg, true); + return 0; +} + +bool checkBufVarData(SFilePage* pPg, const char* varData, uint32_t varLen) { + const char* start = pPg->data + sizeof(SFilePage); + for (uint32_t i = 0; i < (pPg->num - sizeof(SFilePage)) / varLen; ++i) { + if (0 != strncmp(start + 6 * i + 3, varData, varLen - 3)) { + using namespace std; + cout << "pos: " << sizeof(SFilePage) + 6 * i + 3 << " should be " << varData << " but is: " << start + 6 * i + 3 + << endl; + return false; + } + } + return true; +} + +// SPageInfo.pData: | sizeof(void*) 8 bytes | sizeof(SFilePage) 4 bytes| 4096 bytes | +// ^ +// | +// SFilePage: flush to disk from here +void testFlushAndReadBackBuffer() { + SDiskbasedBuf* pBuf = NULL; + uint32_t totalLen = 4096; + auto code = createDiskbasedBuf(&pBuf, totalLen, totalLen * 2, "1", TD_TMP_DIR_PATH); + int32_t pageId = -1; + auto* pPg = (SFilePage*)getNewBufPage(pBuf, &pageId); + ASSERT_TRUE(pPg != nullptr); + pPg->num = sizeof(SFilePage); + + // save data into page + uint32_t len = 6; // sizeof(SFilePage) + 6 * 682 = 4096 + // nullbitmap(1) + len(2) + AA\0(3) + char* rowData = (char*)taosMemoryCalloc(1, len); + *(uint16_t*)(rowData + 2) = (uint16_t)2; + rowData[3] = 'A'; + rowData[4] = 'A'; + + while (pPg->num + len <= getBufPageSize(pBuf)) { + saveDataToPage(pPg, rowData, len); + } + ASSERT_EQ(pPg->num, totalLen); + ASSERT_TRUE(checkBufVarData(pPg, rowData + 3, len)); + releaseBufPage(pBuf, pPg); + + // flush to disk + int32_t newPgId = -1; + pPg = (SFilePage*)getNewBufPage(pBuf, &newPgId); + releaseBufPage(pBuf, pPg); + pPg = (SFilePage*)getNewBufPage(pBuf, &newPgId); + releaseBufPage(pBuf, pPg); + + // reload it from disk + pPg = (SFilePage*)getBufPage(pBuf, pageId); + ASSERT_TRUE(checkBufVarData(pPg, rowData + 3, len)); + destroyDiskbasedBuf(pBuf); +} + } // namespace TEST(testCase, resultBufferTest) { @@ -164,6 +226,7 @@ TEST(testCase, resultBufferTest) { simpleTest(); writeDownTest(); recyclePageTest(); + testFlushAndReadBackBuffer(); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop From 26cbcf996cd3032cdd4d579f1e664643c100bb95 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 May 2023 07:31:14 +0000 Subject: [PATCH 128/238] Merge branch 'enh/3.0' into fix/shared_link_rocksdb --- source/libs/stream/src/streamBackendRocksdb.c | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 793575f59a..e707803d9d 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -747,7 +747,30 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t void** pIter = taosHashIterate(handle->cfInst, NULL); while (*pIter) { RocksdbCfInst* inst = *pIter; - SCfComparator compare = {.comp = inst->pCompares, .numOfComp = cfLen}; + + for (int i = 0; i < cfLen; i++) { + if (inst->cfOpt[i] == NULL) { + rocksdb_options_t* opt = rocksdb_options_create_copy(handle->dbOpt); + rocksdb_block_based_table_options_t* tableOpt = rocksdb_block_based_options_create(); + rocksdb_block_based_options_set_block_cache(tableOpt, handle->cache); + + rocksdb_filterpolicy_t* filter = rocksdb_filterpolicy_create_bloom(15); + rocksdb_block_based_options_set_filter_policy(tableOpt, filter); + + rocksdb_options_set_block_based_table_factory((rocksdb_options_t*)opt, tableOpt); + + SCfInit* cfPara = &ginitDict[i]; + + rocksdb_comparator_t* compare = + rocksdb_comparator_create(NULL, cfPara->detroyFunc, cfPara->cmpFunc, cfPara->cmpName); + rocksdb_options_set_comparator((rocksdb_options_t*)opt, compare); + + inst->pCompares[i] = compare; + inst->cfOpt[i] = opt; + inst->param[i].tableOpt = tableOpt; + } + } + SCfComparator compare = {.comp = inst->pCompares, .numOfComp = cfLen}; inst->pCompareNode = streamBackendAddCompare(handle, &compare); pIter = taosHashIterate(handle->cfInst, pIter); } @@ -897,7 +920,7 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { void streamStateDestroyCompar(void* arg) { SCfComparator* comp = (SCfComparator*)arg; for (int i = 0; i < comp->numOfComp; i++) { - rocksdb_comparator_destroy(comp->comp[i]); + if (comp->comp[i]) rocksdb_comparator_destroy(comp->comp[i]); } taosMemoryFree(comp->comp); } @@ -920,6 +943,8 @@ int streamGetInit(SStreamState* pState, const char* funcName) { char buf[128] = {0}; GEN_COLUMN_FAMILY_NAME(buf, pState->pTdbState->idstr, ginitDict[idx].key); char* err = NULL; + + taosThreadRwlockWrlock(&pState->pTdbState->rwLock); cf = rocksdb_create_column_family(pState->pTdbState->rocksdb, pState->pTdbState->cfOpts[idx], buf, &err); if (err != NULL) { idx = -1; @@ -927,7 +952,6 @@ int streamGetInit(SStreamState* pState, const char* funcName) { funcName, err); taosMemoryFree(err); } - taosThreadRwlockWrlock(&pState->pTdbState->rwLock); pState->pTdbState->pHandle[idx] = cf; taosThreadRwlockUnlock(&pState->pTdbState->rwLock); } From b541a0a053ffb6fc611bf45e53e07b66e2a7c23c Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Mon, 29 May 2023 14:54:58 +0800 Subject: [PATCH 129/238] set sched status --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 77facb0d42..dcfc578ac7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1231,7 +1231,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { streamProcessRunReq(pTask); } else { if (streamTaskShouldPause(&pTask->status)) { - atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__INACTIVE); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); } tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr); } From f2887278e201487c767a26455a5f3a70a4f21ab1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 May 2023 15:44:45 +0800 Subject: [PATCH 130/238] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 14 +- source/dnode/mnode/impl/src/mndScheduler.c | 144 +++++++++++---------- source/dnode/mnode/impl/src/mndStream.c | 2 + source/libs/stream/src/streamTask.c | 4 +- 4 files changed, 87 insertions(+), 77 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 8e7dd0bb0d..7008c0ac40 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -78,11 +78,11 @@ enum { TASK_TRIGGER_STATUS__ACTIVE, }; -enum { +typedef enum { TASK_LEVEL__SOURCE = 1, TASK_LEVEL__AGG, TASK_LEVEL__SINK, -}; +} ETASK_LEVEL; enum { TASK_OUTPUT__FIXED_DISPATCH = 1, @@ -284,13 +284,13 @@ struct SStreamTask { int16_t dispatchMsgType; SStreamStatus status; int32_t selfChildId; - int32_t nodeId; + int32_t nodeId; // vgroup id SEpSet epSet; SCheckpointInfo chkInfo; STaskExec exec; - - // fill history - int8_t fillHistory; + int8_t fillHistory; // fill history + int64_t ekey; // end ts key + int64_t endVer; // end version // children info SArray* childEpInfo; // SArray @@ -351,7 +351,7 @@ typedef struct SStreamMeta { int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); -SStreamTask* tNewStreamTask(int64_t streamId); +SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); void tFreeStreamTask(SStreamTask* pTask); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 734f624be0..18f10e92f9 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -110,6 +110,8 @@ int32_t mndAddSinkToTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask return 0; } +#define SINK_NODE_LEVEL (0) + int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { bool isShuffle = false; @@ -130,15 +132,18 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream if (isShuffle) { memcpy(pTask->shuffleDispatcher.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t sz = taosArrayGetSize(pVgs); - SArray* sinkLv = taosArrayGetP(pStream->tasks, 0); - int32_t sinkLvSize = taosArrayGetSize(sinkLv); - for (int32_t i = 0; i < sz; i++) { + + SArray* pSinkNodes = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); + int32_t numOfSinkNodes = taosArrayGetSize(pSinkNodes); + + int32_t numOfVgroups = taosArrayGetSize(pVgs); + for (int32_t i = 0; i < numOfVgroups; i++) { SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i); - for (int32_t j = 0; j < sinkLvSize; j++) { - SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); - if (pLastLevelTask->nodeId == pVgInfo->vgId) { - pVgInfo->taskId = pLastLevelTask->id.taskId; + + for (int32_t j = 0; j < numOfSinkNodes; j++) { + SStreamTask* pSinkTask = taosArrayGetP(pSinkNodes, j); + if (pSinkTask->nodeId == pVgInfo->vgId) { + pVgInfo->taskId = pSinkTask->id.taskId; break; } } @@ -146,28 +151,33 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream } else { pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; - SArray* pArray = taosArrayGetP(pStream->tasks, 0); + SArray* pSinkNodeList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); + // one sink only - SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); - pTask->fixedEpDispatcher.taskId = lastLevelTask->id.taskId; - pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; - pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; + SStreamTask* lastLevelTask = taosArrayGetP(pSinkNodeList, 0); + STaskDispatcherFixedEp* pDispatcher = &pTask->fixedEpDispatcher; + + pDispatcher->taskId = lastLevelTask->id.taskId; + pDispatcher->nodeId = lastLevelTask->nodeId; + pDispatcher->epSet = lastLevelTask->epSet; } + return 0; } -int32_t mndAssignTaskToVg(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { +int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, const SVgObj* pVgroup) { int32_t msgLen; + pTask->nodeId = pVgroup->vgId; pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); - plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.nodeId = pTask->nodeId; plan->execNode.epSet = pTask->epSet; - if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + return 0; } @@ -210,35 +220,35 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { return pVgroup; } +// create sink node for each vgroup. int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; void* pIter = NULL; - SArray* tasks = taosArrayGetP(pStream->tasks, 0); + SArray* pTaskList = taosArrayGetP(pStream->tasks, 0); while (1) { SVgObj* pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + if (!mndVgroupInDb(pVgroup, pStream->targetDbUid)) { sdbRelease(pSdb, pVgroup); continue; } - SStreamTask* pTask = tNewStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SINK, pStream->fillHistory); if (pTask == NULL) { sdbRelease(pSdb, pVgroup); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pTask->fillHistory = pStream->fillHistory; - mndAddTaskToTaskSet(tasks, pTask); + mndAddTaskToTaskSet(pTaskList, pTask); pTask->nodeId = pVgroup->vgId; pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); - // type - pTask->taskLevel = TASK_LEVEL__SINK; - // sink if (pStream->smaId != 0) { pTask->outputType = TASK_OUTPUT__SMA; @@ -253,6 +263,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } } + sdbRelease(pSdb, pVgroup); } return 0; @@ -260,15 +271,15 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { SArray* tasks = taosArrayGetP(pStream->tasks, 0); - SStreamTask* pTask = tNewStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SINK, pStream->fillHistory); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pTask->fillHistory = pStream->fillHistory; - mndAddTaskToTaskSet(tasks, pTask); + mndAddTaskToTaskSet(tasks, pTask); pTask->nodeId = pStream->fixedSinkVgId; + #if 0 SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId); if (pVgroup == NULL) { @@ -276,9 +287,8 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { } pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); #endif - pTask->epSet = mndGetVgroupEpset(pMnode, &pStream->fixedSinkVg); - pTask->taskLevel = TASK_LEVEL__SINK; + pTask->epSet = mndGetVgroupEpset(pMnode, &pStream->fixedSinkVg); // sink if (pStream->smaId != 0) { @@ -294,16 +304,19 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { return 0; } + + int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { - SSdb* pSdb = pMnode->pSdb; + SSdb* pSdb = pMnode->pSdb; + SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan); if (pPlan == NULL) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans); - pStream->tasks = taosArrayInit(planTotLevel, sizeof(void*)); + int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans); + pStream->tasks = taosArrayInit(planTotLevel, POINTER_BYTES); bool hasExtraSink = false; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; @@ -313,13 +326,13 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - bool multiTarget = pDbObj->cfg.numOfVgroups > 1; + bool multiTarget = (pDbObj->cfg.numOfVgroups > 1); sdbRelease(pSdb, pDbObj); if (planTotLevel == 2 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) { - /*if (true) {*/ - SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); + SArray* taskOneLevel = taosArrayInit(0, POINTER_BYTES); taosArrayPush(pStream->tasks, &taskOneLevel); + // add extra sink hasExtraSink = true; if (pStream->fixedSinkVgId == 0) { @@ -334,6 +347,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } } } + pStream->totalLevel = planTotLevel + hasExtraSink; if (planTotLevel > 1) { @@ -350,22 +364,16 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - pInnerTask = tNewStreamTask(pStream->uid); + pInnerTask = tNewStreamTask(pStream->uid, TASK_LEVEL__AGG, pStream->fillHistory); if (pInnerTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; qDestroyQueryPlan(pPlan); return -1; } - pInnerTask->fillHistory = pStream->fillHistory; mndAddTaskToTaskSet(taskInnerLevel, pInnerTask); - - pInnerTask->childEpInfo = taosArrayInit(0, sizeof(void*)); - - pInnerTask->taskLevel = TASK_LEVEL__AGG; - - // trigger - pInnerTask->triggerParam = pStream->triggerParam; + pInnerTask->childEpInfo = taosArrayInit(0, POINTER_BYTES); + pInnerTask->triggerParam = pStream->triggerParam; // trigger // dispatch if (mndAddDispatcherToInnerTask(pMnode, pStream, pInnerTask) < 0) { @@ -377,7 +385,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); if (pSnode == NULL) { SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignTaskToVg(pMnode, pInnerTask, plan, pVgroup) < 0) { + if (mndAssignStreamTaskToVgroup(pMnode, pInnerTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; @@ -392,11 +400,12 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } } else { SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); - if (mndAssignTaskToVg(pMnode, pInnerTask, plan, pVgroup) < 0) { + if (mndAssignStreamTaskToVgroup(pMnode, pInnerTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; } + sdbRelease(pSdb, pVgroup); } } @@ -422,30 +431,25 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SOURCE, pStream->fillHistory); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; } - pTask->fillHistory = pStream->fillHistory; - mndAddTaskToTaskSet(taskSourceLevel, pTask); + mndAddTaskToTaskSet(taskSourceLevel, pTask); pTask->triggerParam = 0; - // source - pTask->taskLevel = TASK_LEVEL__SOURCE; - - // add fixed vg dispatch - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; + pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; // add fixed vg dispatch pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; pTask->fixedEpDispatcher.taskId = pInnerTask->id.taskId; pTask->fixedEpDispatcher.nodeId = pInnerTask->nodeId; pTask->fixedEpDispatcher.epSet = pInnerTask->epSet; - if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { + if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; @@ -458,6 +462,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { qDestroyQueryPlan(pPlan); return -1; } + pEpInfo->childId = pTask->selfChildId; pEpInfo->epSet = pTask->epSet; pEpInfo->nodeId = pTask->nodeId; @@ -465,10 +470,9 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { taosArrayPush(pInnerTask->childEpInfo, &pEpInfo); sdbRelease(pSdb, pVgroup); } - } - - if (planTotLevel == 1) { - SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); + } else if (planTotLevel == 1) { + // create exec stream task, since only one level, the exec task is also the source task + SArray* taskOneLevel = taosArrayInit(0, POINTER_BYTES); taosArrayPush(pStream->tasks, &taskOneLevel); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); @@ -476,6 +480,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } + SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); if (plan->subplanType != SUBPLAN_TYPE_SCAN) { terrno = TSDB_CODE_QRY_INVALID_INPUT; @@ -486,26 +491,25 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { while (1) { SVgObj* pVgroup; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { sdbRelease(pSdb, pVgroup); continue; } - SStreamTask* pTask = tNewStreamTask(pStream->uid); + // new stream task + SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SOURCE, pStream->fillHistory); if (pTask == NULL) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; } - pTask->fillHistory = pStream->fillHistory; + mndAddTaskToTaskSet(taskOneLevel, pTask); - - // source - pTask->taskLevel = TASK_LEVEL__SOURCE; - - // trigger - pTask->triggerParam = pStream->triggerParam; + pTask->triggerParam = pStream->triggerParam; // trigger // sink or dispatch if (hasExtraSink) { @@ -514,14 +518,16 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { mndAddSinkToTask(pMnode, pStream, pTask); } - if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { + if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; } + sdbRelease(pSdb, pVgroup); } } + qDestroyQueryPlan(pPlan); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0713150b48..39a1fa223f 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -700,6 +700,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { if (pStream->sourceDbUid == streamObj.sourceDbUid) { ++numOfStream; } + sdbRelease(pMnode->pSdb, pStream); if (numOfStream > MND_STREAM_MAX_NUM) { mError("too many streams, no more than %d for each database", MND_STREAM_MAX_NUM); @@ -723,6 +724,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { pDb = NULL; goto _OVER; } + mndReleaseDb(pMnode, pDb); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stream"); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 8a03896978..c57015bc5f 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -17,7 +17,7 @@ #include "tstream.h" #include "wal.h" -SStreamTask* tNewStreamTask(int64_t streamId) { +SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory) { SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { return NULL; @@ -25,6 +25,8 @@ SStreamTask* tNewStreamTask(int64_t streamId) { pTask->id.taskId = tGenIdPI32(); pTask->id.streamId = streamId; + pTask->taskLevel = taskLevel; + pTask->fillHistory = fillHistory; char buf[128] = {0}; sprintf(buf, "0x%" PRIx64 "-%d", pTask->id.streamId, pTask->id.taskId); From 5eb1c559e5553f3d6bff2ab71bfd7e11e6d4d1da Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 15:51:00 +0800 Subject: [PATCH 131/238] fix:save offset if latest --- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/dnode/vnode/src/tq/tqUtil.c | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 17eac7d096..7557ddb4e9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -377,7 +377,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } taosWUnLockLatch(&pTq->lock); - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosMsleep(10); } @@ -410,7 +410,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { while (tqIsHandleExec(pHandle)) { - tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); + tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); taosMsleep(10); } if (pHandle->pRef) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5eaf7b240b..e73aed8966 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -121,6 +121,12 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); + STqOffset offset = {0}; + strcpy(offset.subKey, pRequest->subKey); + if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + return -1; + } } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); From 3a9d55ca35c8a07f3cba456853f4edf0da09e89c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 29 May 2023 15:56:28 +0800 Subject: [PATCH 132/238] fix(rocks/log): set info log level to warn_level --- source/dnode/vnode/src/tsdb/tsdbCache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ef77c9a7c4..6a97ea89b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -128,6 +128,7 @@ static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { rocksdb_options_set_comparator(options, cmp); rocksdb_block_based_options_set_block_cache(tableoptions, cache); rocksdb_options_set_block_based_table_factory(options, tableoptions); + rocksdb_options_set_info_log_level(options, 2); // WARN_LEVEL // rocksdb_options_set_inplace_update_support(options, 1); // rocksdb_options_set_allow_concurrent_memtable_write(options, 0); From f984e67eaa33dd4f5e2fb8ce06d4a97c80b0696b Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 29 May 2023 16:04:35 +0800 Subject: [PATCH 133/238] fix(query): invalid buffer page --- source/libs/executor/inc/executil.h | 1 + source/util/src/tpagedbuf.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 0f169b0c2e..05fdba61d7 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -136,6 +136,7 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); if (!bufPage) { + uFatal("failed to get the buffer page:%d since %s", pos->pageId, terrstr()); return NULL; } if (forUpdate) { diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 218a47ea62..2fb189ec9d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -482,7 +482,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); if (*pInfo != *pi) { - terrno = TSDB_CODE_QRY_SYS_ERROR; + terrno = TSDB_CODE_APP_ERROR; uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id); return NULL; } From 7b8f55c4d6dc9a662e23210cce4556adbd65fa3f Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 29 May 2023 16:10:02 +0800 Subject: [PATCH 134/238] docs: update alter database options for WAL_RETENTION_PERIOD etc. --- docs/zh/12-taos-sql/02-database.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index a2a0914120..b329413aa8 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -121,6 +121,8 @@ alter_database_option: { | WAL_LEVEL value | WAL_FSYNC_PERIOD value | KEEP value + | WAL_RETENTION_PERIOD value + | WAL_RETENTION_SIZE value } ``` From 23983e506fae3c7ff7a2c168296aed3a01050d94 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 29 May 2023 16:12:26 +0800 Subject: [PATCH 135/238] fix test case --- source/dnode/mnode/impl/src/mndDnode.c | 12 +++++------- tests/script/tsim/dnode/drop_dnode_has_mnode.sim | 2 +- tests/script/tsim/dnode/offline_reason.sim | 2 +- tests/script/tsim/valgrind/checkError1.sim | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c4b0e00c76..73dbb243a1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -995,13 +995,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) { - if (!isonline && !force) { - terrno = TSDB_CODE_DNODE_OFFLINE; - mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), - numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); - goto _OVER; - } + if (!isonline && !force) { + terrno = TSDB_CODE_DNODE_OFFLINE; + mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), + numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); + goto _OVER; } code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes, force, dropReq.unsafe); diff --git a/tests/script/tsim/dnode/drop_dnode_has_mnode.sim b/tests/script/tsim/dnode/drop_dnode_has_mnode.sim index 054f978607..8a74367726 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_mnode.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_mnode.sim @@ -35,7 +35,7 @@ endi print =============== step2 drop dnode 3 sql_error drop dnode 1 -sql drop dnode 3 +sql drop dnode 3 force sql select * from information_schema.ins_dnodes print ===> $data00 $data01 $data02 $data03 $data04 $data05 diff --git a/tests/script/tsim/dnode/offline_reason.sim b/tests/script/tsim/dnode/offline_reason.sim index 8c4d8b47f7..23f015392b 100644 --- a/tests/script/tsim/dnode/offline_reason.sim +++ b/tests/script/tsim/dnode/offline_reason.sim @@ -57,7 +57,7 @@ if $data(2)[7] != @status msg timeout@ then endi print ========== step4 -sql drop dnode 2 +sql drop dnode 2 force sql select * from information_schema.ins_dnodes if $rows != 1 then return -1 diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 5f82d2d935..debe633f06 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -20,7 +20,7 @@ sql_error alter user u2 sysinfo 0 print =============== step2 create drop dnode sql create dnode $hostname port 7200 sql create dnode $hostname port 7300 -sql drop dnode 3 +sql drop dnode 3 force sql alter dnode 1 'debugflag 131' print =============== step3: select * from information_schema.ins_dnodes From ccf57129bb6cadda1307b528012ab066ba474eeb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 29 May 2023 16:33:31 +0800 Subject: [PATCH 136/238] fix: show create table privilege issue --- source/libs/catalog/src/ctgUtil.c | 4 ++-- source/libs/parser/src/parAstParser.c | 2 +- source/libs/parser/src/parAuthenticator.c | 2 +- source/libs/parser/src/parUtil.c | 11 +++++++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index ae8fe0cca4..ca63f794e7 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1482,7 +1482,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { switch (pReq->type) { case AUTH_TYPE_READ: { - if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { req->singleType = AUTH_TYPE_READ; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { @@ -1498,7 +1498,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { break; } case AUTH_TYPE_WRITE: { - if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { req->singleType = AUTH_TYPE_WRITE; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5db1f5dbdb..df080b574c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -581,7 +581,7 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_READ, + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, pCxt->pMetaCache); } return code; diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 1586d8128b..251d3bd0cb 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -175,7 +175,7 @@ static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) { static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) { SNode* pTagCond = NULL; // todo check tag condition for subtable - return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_READ, &pTagCond); + return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, &pTagCond); } static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 14da6f8aab..9c95e2b17c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -498,7 +498,7 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, char* pStr) { - return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL != pTable && '\0' == pTable[0]) ? NULL : pTable, + return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, type); } @@ -524,6 +524,9 @@ static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { strncpy(pStr, pStart, p - pStart); *pNext = ++p; } + if (*pStart == '`' && *(pStart + 1) == '`') { + *pStr = 0; + } } static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) { @@ -532,7 +535,11 @@ static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUser pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p); getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p); getStringFromAuthStr(p, pUserAuth->tbName.tname, &p); - pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + if (pUserAuth->tbName.tname[0]) { + pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + } else { + pUserAuth->tbName.type = TSDB_DB_NAME_T; + } pUserAuth->type = getIntegerFromAuthStr(p, &p); } From 7977490b81d25dcfec8fc1e5a71aad37ecc78096 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 May 2023 08:42:52 +0000 Subject: [PATCH 137/238] delete geos opt --- cmake/cmake.options | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index 555b72cbdf..b00ae14715 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -80,7 +80,7 @@ ENDIF () option( BUILD_GEOS "If build geos on Windows" - ON + OFF ) option( From 9eb15e41efc8a3b936746151163018420bf17b48 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 16:44:03 +0800 Subject: [PATCH 138/238] fix:put poll to push manager if wal not exist when offset is latest --- source/dnode/vnode/src/tq/tqUtil.c | 40 ++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index e73aed8966..d30ed638f1 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -87,12 +87,13 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { return 0; } -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest) { +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, bool* pBlockReturned) { uint64_t consumerId = pRequest->consumerId; STqOffsetVal reqOffset = pRequest->reqOffset; STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); int32_t vgId = TD_VID(pTq->pVnode); + *pBlockReturned = false; // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. if (pOffset != NULL) { *pOffsetVal = pOffset->val; @@ -120,12 +121,27 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); - tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); - STqOffset offset = {0}; - strcpy(offset.subKey, pRequest->subKey); - if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { - terrno = TSDB_CODE_PAR_INTERNAL_ERROR; - return -1; + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); + + tqOffsetResetToLog(&dataRsp.rspOffset, pHandle->pRef->refVer); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, + pHandle->subKey, vgId, dataRsp.rspOffset.version); + int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + tDeleteSMqDataRsp(&dataRsp); + + *pBlockReturned = true; + return code; + } else { + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, pRequest); + tqOffsetResetToLog(&taosxRsp.rspOffset, pHandle->pRef->refVer); + int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); + tDeleteSTaosxRsp(&taosxRsp); + + *pBlockReturned = true; + return code; } } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", @@ -154,8 +170,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST || (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId)) { + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) { // lock taosWLockLatch(&pTq->lock); code = tqRegisterPushHandle(pTq, pHandle, pMsg); @@ -296,11 +311,16 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ // 1. reset the offset if needed if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { // handle the reset offset cases, according to the consumer's choice. - code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest); + bool blockReturned = false; + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); if (code != 0) { return code; } + // empty block returned, quit + if (blockReturned) { + return 0; + } } else { // use the consumer specified offset // the offset value can not be monotonious increase?? offset = reqOffset; From c2bb44b6877a9953735a5f990366badc8ad551c8 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Mon, 29 May 2023 16:48:57 +0800 Subject: [PATCH 139/238] fix: TDengine build on macOS arm64 (#21513) --- cmake/cmake.platform | 12 ++++++++++-- contrib/CMakeLists.txt | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index cb09bf2085..f9faf7316c 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -56,9 +56,17 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(TD_DARWIN TRUE) SET(OSTYPE "macOS") + execute_process(COMMAND geos-config --cflags OUTPUT_VARIABLE GEOS_CFLAGS) + execute_process(COMMAND geos-config --ldflags OUTPUT_VARIABLE GEOS_LDFLAGS) + string(SUBSTRING ${GEOS_CFLAGS} 2 -1 GEOS_CFLAGS) + string(REGEX REPLACE "\n" "" GEOS_CFLAGS ${GEOS_CFLAGS}) + string(SUBSTRING ${GEOS_LDFLAGS} 2 -1 GEOS_LDFLAGS) + string(REGEX REPLACE "\n" "" GEOS_LDFLAGS ${GEOS_LDFLAGS}) + MESSAGE("GEOS_CFLAGS "${GEOS_CFLAGS}) + MESSAGE("GEOS_LDFLAGS "${GEOS_LDFLAGS}) ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") - INCLUDE_DIRECTORIES(/usr/local/include) - LINK_DIRECTORIES(/usr/local/lib) + INCLUDE_DIRECTORIES(${GEOS_CFLAGS}) + LINK_DIRECTORIES(${GEOS_LDFLAGS}) IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") MESSAGE("Current system arch is arm64") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b3333dee99..eeb20ee884 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -236,6 +236,10 @@ if(${BUILD_WITH_ROCKSDB}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") endif(${TD_DARWIN}) + if (${TD_DARWIN_ARM64}) + set(HAS_ARMV8_CRC true) + endif(${TD_DARWIN_ARM64}) + if (${TD_WINDOWS}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") endif(${TD_WINDOWS}) From 07bdb95b1a36f8b999d5466dee54d6d21cdc58df Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 17:10:11 +0800 Subject: [PATCH 140/238] fix:compile error --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqMeta.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/executor/src/executil.c | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 5225a7a644..66418e6841 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -82,7 +82,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); -int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* node, SArray **tableList); +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList); /** * set the task Id, usually used by message queue process diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fb9f620c20..01a60a99e4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -713,7 +713,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } SArray* tbUidList = NULL; - ret = qGetTableList(req.suid, pVnode->pMeta, pVnode, pHandle->execHandle.execTb.node, &tbUidList); + ret = qGetTableList(req.suid, pVnode, pHandle->execHandle.execTb.node, &tbUidList); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d vgId:%d, subkey:%s consumer:0x%" PRIx64, ret, pVnode->config.vgId, req.subKey, pHandle->consumerId); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 32f4931fae..d4371f1264 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -347,7 +347,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { } SArray* tbUidList = NULL; - int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList); + int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle.subKey, handle.consumerId); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index fea5104017..a5ec0d4def 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1084,7 +1084,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* list = NULL; - int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode->pMeta, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list); + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); taosArrayDestroy(list); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index e359cf51e3..19b64d9833 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1152,14 +1152,14 @@ _end: return code; } -int32_t qGetTableList(int64_t suid, void* metaHandle, void* pVnode, void* node, SArray **tableList){ +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList){ SSubplan *pSubplan = (SSubplan *)node; SScanPhysiNode pNode = {0}; pNode.suid = suid; pNode.uid = suid; pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); - int code = getTableList(metaHandle, pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList"); + int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList"); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From 41ed0138b437630f0b6cd0d08701c1ad0e85463c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 17:25:11 +0800 Subject: [PATCH 141/238] fix:compile error --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 10 +++++----- source/dnode/vnode/src/tq/tqMeta.c | 9 ++++----- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/libs/executor/src/executil.c | 4 ++-- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 66418e6841..e46757f6f8 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -82,7 +82,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id); -int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList); +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList, void* pTaskInfo); /** * set the task Id, usually used by message queue process diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 01a60a99e4..f8116f49ef 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -712,8 +712,12 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } } + buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, + (SSnapContext**)(&handle.sContext)); + pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); + SArray* tbUidList = NULL; - ret = qGetTableList(req.suid, pVnode, pHandle->execHandle.execTb.node, &tbUidList); + ret = qGetTableList(req.suid, pVnode, pHandle->execHandle.execTb.node, &tbUidList, pHandle->execHandle.task); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d vgId:%d, subkey:%s consumer:0x%" PRIx64, ret, pVnode->config.vgId, req.subKey, pHandle->consumerId); taosArrayDestroy(tbUidList); @@ -723,10 +727,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); - - buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, - (SSnapContext**)(&handle.sContext)); - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); } taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index d4371f1264..0edffd7f05 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -345,9 +345,12 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { return -1; } } + buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, + handle.fetchMeta, (SSnapContext**)(&reader.sContext)); + handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); SArray* tbUidList = NULL; - int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList); + int ret = qGetTableList(handle.execHandle.execTb.suid, pTq->pVnode, handle.execHandle.execTb.node, &tbUidList, handle.execHandle.task); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList error:%d handle %s consumer:0x%" PRIx64, ret, handle.subKey, handle.consumerId); taosArrayDestroy(tbUidList); @@ -357,10 +360,6 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); - - buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, - handle.fetchMeta, (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); taosWLockLatch(&pTq->lock); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index a5ec0d4def..6ef232995a 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1084,7 +1084,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* list = NULL; - int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list); + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list, pTqHandle->execHandle.task); if(ret != TDB_CODE_SUCCESS) { tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); taosArrayDestroy(list); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 19b64d9833..a168050645 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1152,14 +1152,14 @@ _end: return code; } -int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList){ +int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList, void* pTaskInfo){ SSubplan *pSubplan = (SSubplan *)node; SScanPhysiNode pNode = {0}; pNode.suid = suid; pNode.uid = suid; pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); - int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList"); + int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From 16e14b1c828a75a6ad739b41d601b0dd34acb795 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 17:27:35 +0800 Subject: [PATCH 142/238] fix:compile error --- source/libs/executor/src/executil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a168050645..f4531729c4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1159,7 +1159,8 @@ int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList pNode.uid = suid; pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); - int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); + uint8_t digest[17] = {0}; + int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, digest, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From cd2d225c933505e8599e4f630116bbcfaeb00b8d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 May 2023 17:50:39 +0800 Subject: [PATCH 143/238] refactor: do some internal refactor. --- source/dnode/mnode/impl/inc/mndDef.h | 3 +- source/dnode/mnode/impl/src/mndScheduler.c | 247 ++++++++++----------- source/libs/stream/src/streamTask.c | 1 + 3 files changed, 120 insertions(+), 131 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 7d707f4cba..82b714e6eb 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -663,9 +663,10 @@ typedef struct { char targetDb[TSDB_DB_FNAME_LEN]; char targetSTbName[TSDB_TABLE_FNAME_LEN]; int64_t targetStbUid; - int32_t fixedSinkVgId; // 0 for shuffle + // fixedSinkVg is not applicable for encode and decode SVgObj fixedSinkVg; + int32_t fixedSinkVgId; // 0 for shuffle // transformation char* sql; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 18f10e92f9..b68565b146 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -14,18 +14,8 @@ */ #include "mndScheduler.h" -#include "mndConsumer.h" #include "mndDb.h" -#include "mndDnode.h" -#include "mndMnode.h" -#include "mndShow.h" #include "mndSnode.h" -#include "mndStb.h" -#include "mndStream.h" -#include "mndSubscribe.h" -#include "mndTopic.h" -#include "mndTrans.h" -#include "mndUser.h" #include "mndVgroup.h" #include "parser.h" #include "tcompare.h" @@ -34,7 +24,10 @@ extern bool tsDeployOnSnode; -static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) { +static int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup); +static void setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask); + +static int32_t mndAddToTaskset(SArray* pArray, SStreamTask* pTask) { int32_t childId = taosArrayGetSize(pArray); pTask->selfChildId = childId; taosArrayPush(pArray, &pTask); @@ -97,7 +90,7 @@ END: return terrno; } -int32_t mndAddSinkToTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { +int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) { if (pStream->smaId != 0) { pTask->outputType = TASK_OUTPUT__SMA; pTask->smaSink.smaId = pStream->smaId; @@ -106,18 +99,23 @@ int32_t mndAddSinkToTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask pTask->tbSink.stbUid = pStream->targetStbUid; memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); + if (pTask->tbSink.pSchemaWrapper == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } + return 0; } #define SINK_NODE_LEVEL (0) -int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { +int32_t mndAddDispatcherForInnerTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { bool isShuffle = false; if (pStream->fixedSinkVgId == 0) { SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); if (pDb != NULL && pDb->cfg.numOfVgroups > 1) { + isShuffle = true; pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; @@ -129,19 +127,19 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream sdbRelease(pMnode->pSdb, pDb); } + SArray* pSinkNodeList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); + int32_t numOfSinkNodes = taosArrayGetSize(pSinkNodeList); + if (isShuffle) { memcpy(pTask->shuffleDispatcher.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - SArray* pSinkNodes = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); - int32_t numOfSinkNodes = taosArrayGetSize(pSinkNodes); - int32_t numOfVgroups = taosArrayGetSize(pVgs); for (int32_t i = 0; i < numOfVgroups; i++) { SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i); for (int32_t j = 0; j < numOfSinkNodes; j++) { - SStreamTask* pSinkTask = taosArrayGetP(pSinkNodes, j); + SStreamTask* pSinkTask = taosArrayGetP(pSinkNodeList, j); if (pSinkTask->nodeId == pVgInfo->vgId) { pVgInfo->taskId = pSinkTask->id.taskId; break; @@ -149,17 +147,8 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream } } } else { - pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; - SArray* pSinkNodeList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); - - // one sink only - SStreamTask* lastLevelTask = taosArrayGetP(pSinkNodeList, 0); - STaskDispatcherFixedEp* pDispatcher = &pTask->fixedEpDispatcher; - - pDispatcher->taskId = lastLevelTask->id.taskId; - pDispatcher->nodeId = lastLevelTask->nodeId; - pDispatcher->epSet = lastLevelTask->epSet; + SStreamTask* pOneSinkTask = taosArrayGetP(pSinkNodeList, 0); + setFixedDownstreamEpInfo(pTask, pOneSinkTask); } return 0; @@ -224,7 +213,6 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) { int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; void* pIter = NULL; - SArray* pTaskList = taosArrayGetP(pStream->tasks, 0); while (1) { SVgObj* pVgroup = NULL; @@ -238,73 +226,97 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SINK, pStream->fillHistory); - if (pTask == NULL) { - sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - mndAddTaskToTaskSet(pTaskList, pTask); - pTask->nodeId = pVgroup->vgId; - pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); - - // sink - if (pStream->smaId != 0) { - pTask->outputType = TASK_OUTPUT__SMA; - pTask->smaSink.smaId = pStream->smaId; - } else { - pTask->outputType = TASK_OUTPUT__TABLE; - pTask->tbSink.stbUid = pStream->targetStbUid; - memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); - pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); - if (pTask->tbSink.pSchemaWrapper == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - } - + mndAddSinkTaskToStream(pStream, pMnode, pVgroup->vgId, pVgroup); sdbRelease(pSdb, pVgroup); } + return 0; } -int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { - SArray* tasks = taosArrayGetP(pStream->tasks, 0); +int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup) { + SArray* pTaskList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); + SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SINK, pStream->fillHistory); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - mndAddTaskToTaskSet(tasks, pTask); - pTask->nodeId = pStream->fixedSinkVgId; + mndAddToTaskset(pTaskList, pTask); -#if 0 - SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId); - if (pVgroup == NULL) { - return -1; - } + pTask->nodeId = vgId; pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); -#endif - - pTask->epSet = mndGetVgroupEpset(pMnode, &pStream->fixedSinkVg); - - // sink - if (pStream->smaId != 0) { - pTask->outputType = TASK_OUTPUT__SMA; - pTask->smaSink.smaId = pStream->smaId; - } else { - pTask->outputType = TASK_OUTPUT__TABLE; - pTask->tbSink.stbUid = pStream->targetStbUid; - memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); - pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); - } - + mndSetSinkTaskInfo(pStream, pTask); return 0; } +static int32_t mndScheduleFillHistoryStreamTask(SMnode* pMnode, SStreamObj* pStream) { + return 0; +} +static int32_t addSourceStreamTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, SStreamObj* pStream, + SSubplan* plan, uint64_t uid, int8_t taskLevel, int8_t fillHistory, + bool hasExtraSink) { + SStreamTask* pTask = tNewStreamTask(uid, taskLevel, fillHistory); + if (pTask == NULL) { + return terrno; + } + + mndAddToTaskset(pTaskList, pTask); + pTask->triggerParam = pStream->triggerParam; // trigger + + // sink or dispatch + if (hasExtraSink) { + mndAddDispatcherForInnerTask(pMnode, pStream, pTask); + } else { + mndSetSinkTaskInfo(pStream, pTask); + } + + if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) { + return terrno; + } + + return TSDB_CODE_SUCCESS; +} + +static SStreamChildEpInfo* createStreamTaskEpInfo(SStreamTask* pTask) { + SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo)); + if (pEpInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pEpInfo->childId = pTask->selfChildId; + pEpInfo->epSet = pTask->epSet; + pEpInfo->nodeId = pTask->nodeId; + pEpInfo->taskId = pTask->id.taskId; + + return pEpInfo; +} + +void setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask) { + STaskDispatcherFixedEp* pDispatcher = &pDstTask->fixedEpDispatcher; + pDispatcher->taskId = pTask->id.taskId; + pDispatcher->nodeId = pTask->nodeId; + pDispatcher->epSet = pTask->epSet; + + pDstTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; + pDstTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; +} + +int32_t appendToUpstream(SStreamTask* pTask, SStreamTask* pUpstream) { + SStreamChildEpInfo* pEpInfo = createStreamTaskEpInfo(pTask); + if (pEpInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if(pUpstream->childEpInfo == NULL) { + pUpstream->childEpInfo = taosArrayInit(4, POINTER_BYTES); + } + + taosArrayPush(pUpstream->childEpInfo, &pEpInfo); + return TSDB_CODE_SUCCESS; +} int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { SSdb* pSdb = pMnode->pSdb; @@ -341,7 +353,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } } else { - if (mndAddFixedSinkTaskToStream(pMnode, pStream) < 0) { + if (mndAddSinkTaskToStream(pStream, pMnode, pStream->fixedSinkVgId, &pStream->fixedSinkVg) < 0) { // TODO free return -1; } @@ -354,7 +366,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { SStreamTask* pInnerTask; // inner level { - SArray* taskInnerLevel = taosArrayInit(0, sizeof(void*)); + SArray* taskInnerLevel = taosArrayInit(0, POINTER_BYTES); taosArrayPush(pStream->tasks, &taskInnerLevel); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); @@ -371,12 +383,11 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - mndAddTaskToTaskSet(taskInnerLevel, pInnerTask); - pInnerTask->childEpInfo = taosArrayInit(0, POINTER_BYTES); + mndAddToTaskset(taskInnerLevel, pInnerTask); pInnerTask->triggerParam = pStream->triggerParam; // trigger // dispatch - if (mndAddDispatcherToInnerTask(pMnode, pStream, pInnerTask) < 0) { + if (mndAddDispatcherForInnerTask(pMnode, pStream, pInnerTask) < 0) { qDestroyQueryPlan(pPlan); return -1; } @@ -411,7 +422,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } // source level - SArray* taskSourceLevel = taosArrayInit(0, sizeof(void*)); + SArray* taskSourceLevel = taosArrayInit(0, POINTER_BYTES); taosArrayPush(pStream->tasks, &taskSourceLevel); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1); @@ -425,7 +436,10 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { while (1) { SVgObj* pVgroup; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) { sdbRelease(pSdb, pVgroup); continue; @@ -439,41 +453,31 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - mndAddTaskToTaskSet(taskSourceLevel, pTask); + mndAddToTaskset(taskSourceLevel, pTask); + + // all the source tasks dispatch result to a single agg node. + setFixedDownstreamEpInfo(pTask, pInnerTask); pTask->triggerParam = 0; - pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; // add fixed vg dispatch - pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; - - pTask->fixedEpDispatcher.taskId = pInnerTask->id.taskId; - pTask->fixedEpDispatcher.nodeId = pInnerTask->nodeId; - pTask->fixedEpDispatcher.epSet = pInnerTask->epSet; - if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); return -1; } - SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo)); - if (pEpInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - sdbRelease(pSdb, pVgroup); + int32_t code = appendToUpstream(pTask, pInnerTask); + sdbRelease(pSdb, pVgroup); + + if (code != TSDB_CODE_SUCCESS) { + terrno = code; qDestroyQueryPlan(pPlan); return -1; } - - pEpInfo->childId = pTask->selfChildId; - pEpInfo->epSet = pTask->epSet; - pEpInfo->nodeId = pTask->nodeId; - pEpInfo->taskId = pTask->id.taskId; - taosArrayPush(pInnerTask->childEpInfo, &pEpInfo); - sdbRelease(pSdb, pVgroup); } } else if (planTotLevel == 1) { // create exec stream task, since only one level, the exec task is also the source task - SArray* taskOneLevel = taosArrayInit(0, POINTER_BYTES); - taosArrayPush(pStream->tasks, &taskOneLevel); + SArray* pTaskList = taosArrayInit(0, POINTER_BYTES); + taosArrayPush(pStream->tasks, &pTaskList); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); if (LIST_LENGTH(inner->pNodeList) != 1) { @@ -501,30 +505,13 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } // new stream task - SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SOURCE, pStream->fillHistory); - if (pTask == NULL) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } - - mndAddTaskToTaskSet(taskOneLevel, pTask); - pTask->triggerParam = pStream->triggerParam; // trigger - - // sink or dispatch - if (hasExtraSink) { - mndAddDispatcherToInnerTask(pMnode, pStream, pTask); - } else { - mndAddSinkToTask(pMnode, pStream, pTask); - } - - if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - return -1; - } - + int32_t code = addSourceStreamTask(pMnode, pVgroup, pTaskList, pStream, plan, pStream->uid, TASK_LEVEL__SOURCE, pStream->fillHistory, hasExtraSink); sdbRelease(pSdb, pVgroup); + + if (code != TSDB_CODE_SUCCESS) { + qDestroyQueryPlan(pPlan); + return -1; + } } } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index c57015bc5f..5152e43dbd 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -20,6 +20,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory) { SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } From 3628888ecc69195c5ecafe955e5810b5008636ce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 29 May 2023 18:02:45 +0800 Subject: [PATCH 144/238] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 2 +- source/dnode/mnode/impl/src/mndScheduler.c | 26 ++++------------------ source/libs/stream/src/streamTask.c | 11 ++++++++- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 7008c0ac40..1d4bbf073e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -351,7 +351,7 @@ typedef struct SStreamMeta { int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); -SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory); +SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, SArray* pTaskList); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); void tFreeStreamTask(SStreamTask* pTask); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index b68565b146..df8c11a6f6 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -27,13 +27,6 @@ extern bool tsDeployOnSnode; static int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup); static void setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask); -static int32_t mndAddToTaskset(SArray* pArray, SStreamTask* pTask) { - int32_t childId = taosArrayGetSize(pArray); - pTask->selfChildId = childId; - taosArrayPush(pArray, &pTask); - return 0; -} - int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, int64_t deleteMark) { SNode* pAst = NULL; @@ -236,14 +229,12 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { int32_t mndAddSinkTaskToStream(SStreamObj* pStream, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup) { SArray* pTaskList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL); - SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SINK, pStream->fillHistory); + SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SINK, pStream->fillHistory, 0, pTaskList); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - mndAddToTaskset(pTaskList, pTask); - pTask->nodeId = vgId; pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); mndSetSinkTaskInfo(pStream, pTask); @@ -257,14 +248,11 @@ static int32_t mndScheduleFillHistoryStreamTask(SMnode* pMnode, SStreamObj* pStr static int32_t addSourceStreamTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, SStreamObj* pStream, SSubplan* plan, uint64_t uid, int8_t taskLevel, int8_t fillHistory, bool hasExtraSink) { - SStreamTask* pTask = tNewStreamTask(uid, taskLevel, fillHistory); + SStreamTask* pTask = tNewStreamTask(uid, taskLevel, fillHistory, pStream->triggerParam, pTaskList); if (pTask == NULL) { return terrno; } - mndAddToTaskset(pTaskList, pTask); - pTask->triggerParam = pStream->triggerParam; // trigger - // sink or dispatch if (hasExtraSink) { mndAddDispatcherForInnerTask(pMnode, pStream, pTask); @@ -376,16 +364,13 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - pInnerTask = tNewStreamTask(pStream->uid, TASK_LEVEL__AGG, pStream->fillHistory); + pInnerTask = tNewStreamTask(pStream->uid, TASK_LEVEL__AGG, pStream->fillHistory, pStream->triggerParam, taskInnerLevel); if (pInnerTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; qDestroyQueryPlan(pPlan); return -1; } - mndAddToTaskset(taskInnerLevel, pInnerTask); - pInnerTask->triggerParam = pStream->triggerParam; // trigger - // dispatch if (mndAddDispatcherForInnerTask(pMnode, pStream, pInnerTask) < 0) { qDestroyQueryPlan(pPlan); @@ -445,7 +430,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SOURCE, pStream->fillHistory); + SStreamTask* pTask = tNewStreamTask(pStream->uid, TASK_LEVEL__SOURCE, pStream->fillHistory, 0, taskSourceLevel); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); @@ -453,11 +438,8 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - mndAddToTaskset(taskSourceLevel, pTask); - // all the source tasks dispatch result to a single agg node. setFixedDownstreamEpInfo(pTask, pInnerTask); - pTask->triggerParam = 0; if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 5152e43dbd..a0caffd41f 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -17,7 +17,14 @@ #include "tstream.h" #include "wal.h" -SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory) { +static int32_t mndAddToTaskset(SArray* pArray, SStreamTask* pTask) { + int32_t childId = taosArrayGetSize(pArray); + pTask->selfChildId = childId; + taosArrayPush(pArray, &pTask); + return 0; +} + +SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, SArray* pTaskList) { SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -28,6 +35,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHisto pTask->id.streamId = streamId; pTask->taskLevel = taskLevel; pTask->fillHistory = fillHistory; + pTask->triggerParam = triggerParam; char buf[128] = {0}; sprintf(buf, "0x%" PRIx64 "-%d", pTask->id.streamId, pTask->id.taskId); @@ -37,6 +45,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHisto pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; + mndAddToTaskset(pTaskList, pTask); return pTask; } From e00f79367dfe02ad70b2ec4705c7d40b6e0ab18e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 29 May 2023 18:57:21 +0800 Subject: [PATCH 145/238] merge 3.0 --- source/libs/stream/src/streamBackendRocksdb.c | 72 +++++++++++-------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index e707803d9d..f7638a42ae 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -142,16 +142,17 @@ void streamBackendCleanup(void* arg) { } taosHashCleanup(pHandle->cfInst); - rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); - char* err = NULL; - rocksdb_flush(pHandle->db, flushOpt, &err); - if (err != NULL) { - qError("failed to flush db before streamBackend clean up, reason:%s", err); - taosMemoryFree(err); + if (pHandle->db) { + char* err = NULL; + rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); + rocksdb_flush(pHandle->db, flushOpt, &err); + if (err != NULL) { + qError("failed to flush db before streamBackend clean up, reason:%s", err); + taosMemoryFree(err); + } + rocksdb_flushoptions_destroy(flushOpt); + rocksdb_close(pHandle->db); } - rocksdb_flushoptions_destroy(flushOpt); - - rocksdb_close(pHandle->db); rocksdb_options_destroy(pHandle->dbOpt); rocksdb_env_destroy(pHandle->env); rocksdb_cache_destroy(pHandle->cache); @@ -650,7 +651,7 @@ rocksdb_compactionfilter_t* compactFilteFactoryCreateFilter(void* arg, rocksdb_c void destroyRocksdbCfInst(RocksdbCfInst* inst) { int cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); for (int i = 0; i < cfLen; i++) { - rocksdb_column_family_handle_destroy(inst->pHandle[i]); + if (inst->pHandle[i]) rocksdb_column_family_handle_destroy((inst->pHandle)[i]); } rocksdb_writeoptions_destroy(inst->wOpt); @@ -668,7 +669,6 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t int64_t streamId; int32_t taskId, dummy = 0; char suffix[64] = {0}; - SHashObj* instTbl = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); rocksdb_options_t** cfOpts = taosMemoryCalloc(nCf, sizeof(rocksdb_options_t*)); RocksdbCfParam* params = taosMemoryCalloc(nCf, sizeof(RocksdbCfParam*)); @@ -705,8 +705,12 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t qError("failed to open rocksdb cf, reason:%s", err); taosMemoryFree(err); } else { - qDebug("succ to open rocksdb cf, reason"); + qDebug("succ to open rocksdb cf"); } + // close default cf + rocksdb_column_family_handle_destroy(cfHandle[0]); + rocksdb_options_destroy(cfOpts[0]); + handle->db = db; static int32_t cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]); for (int i = 0; i < nCf; i++) { @@ -720,7 +724,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t int idx = streamGetInit(NULL, funcname); RocksdbCfInst* inst = NULL; - RocksdbCfInst** pInst = taosHashGet(instTbl, idstr, strlen(idstr) + 1); + RocksdbCfInst** pInst = taosHashGet(handle->cfInst, idstr, strlen(idstr) + 1); if (pInst == NULL || *pInst == NULL) { inst = taosMemoryCalloc(1, sizeof(RocksdbCfInst)); inst->pHandle = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); @@ -745,7 +749,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t } } void** pIter = taosHashIterate(handle->cfInst, NULL); - while (*pIter) { + while (pIter) { RocksdbCfInst* inst = *pIter; for (int i = 0; i < cfLen; i++) { @@ -831,7 +835,7 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { rocksdb_options_set_comparator((rocksdb_options_t*)cfOpt[i], compare); pCompare[i] = compare; } - rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(1, cfLen * sizeof(rocksdb_column_family_handle_t*)); + rocksdb_column_family_handle_t** cfHandle = taosMemoryCalloc(cfLen, sizeof(rocksdb_column_family_handle_t*)); pState->pTdbState->rocksdb = handle->db; pState->pTdbState->pHandle = (void**)cfHandle; pState->pTdbState->writeOpts = rocksdb_writeoptions_create(); @@ -873,7 +877,8 @@ void streamStateCloseBackend(SStreamState* pState, bool remove) { if (remove) { for (int i = 0; i < cfLen; i++) { if (pState->pTdbState->pHandle[i] != NULL) - rocksdb_drop_column_family(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[i], &err); + rocksdb_drop_column_family(pState->pTdbState->rocksdb, + ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[i], &err); if (err != NULL) { qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); taosMemoryFreeClear(err); @@ -951,8 +956,9 @@ int streamGetInit(SStreamState* pState, const char* funcName) { qError("failed to to open cf, %p 0x%" PRIx64 "-%d_%s, reason:%s", pState, pState->streamId, pState->taskId, funcName, err); taosMemoryFree(err); + } else { + pState->pTdbState->pHandle[idx] = cf; } - pState->pTdbState->pHandle[idx] = cf; taosThreadRwlockUnlock(&pState->pTdbState->rwLock); } } @@ -982,7 +988,8 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa rocksdb_readoptions_set_snapshot(rOpt, *snapshot); rocksdb_readoptions_set_fill_cache(rOpt, 0); - return rocksdb_create_iterator_cf(pState->pTdbState->rocksdb, rOpt, pState->pTdbState->pHandle[idx]); + return rocksdb_create_iterator_cf(pState->pTdbState->rocksdb, rOpt, + ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[idx]); } #define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ @@ -999,11 +1006,12 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ - char* ttlV = NULL; \ - int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ + rocksdb_column_family_handle_t* pHandle = \ + ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = pState->pTdbState->rocksdb; \ + rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ + char* ttlV = NULL; \ + int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ if (err != NULL) { \ taosMemoryFree(err); \ @@ -1029,11 +1037,12 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_readoptions_t* opts = pState->pTdbState->readOpts; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + rocksdb_column_family_handle_t* pHandle = \ + ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = pState->pTdbState->rocksdb; \ + rocksdb_readoptions_t* opts = pState->pTdbState->readOpts; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ if (val == NULL) { \ if (err == NULL) { \ qDebug("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ @@ -1076,9 +1085,10 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa char toString[128] = {0}; \ if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ + rocksdb_column_family_handle_t* pHandle = \ + ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = pState->pTdbState->rocksdb; \ + rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ rocksdb_delete_cf(db, opts, pHandle, (const char*)buf, klen, &err); \ if (err != NULL) { \ qError("streamState str: %s failed to del from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ From 539b138906aea150510641ce087fba6a1463f1da Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Mon, 29 May 2023 19:23:45 +0800 Subject: [PATCH 146/238] Update 01-faq.md --- docs/zh/27-train-faq/01-faq.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index bf46f3ca1f..b5501d8915 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -247,10 +247,14 @@ launchctl limit maxfiles 该提示是创建 db 的 vnode 数量不够了，需要的 vnode 不能超过了 dnode 中 vnode 的上限。因为系统默认是一个 dnode 中有 CPU 核数两倍的 vnode，也可以通过配置文件中的参数 supportVnodes 控制。 正常调大 taos.cfg 中 supportVnodes 参数即可。 -### 21 【查询】在服务器上的使用 tao-CLI 能查到指定时间段的数据，但在客户端机器上查不到？ +### 21 在服务器上的使用 taos-CLI 能查到指定时间段的数据，但在客户端机器上查不到？ 这种情况是因为客户端与服务器上设置的时区不一致导致的，调整客户端与服务器的时区一致即可解决。 -### 22 【表名】表名确认是存在的，但写入或查询时报表不存在错误，非常奇怪，什么原因？ +### 22 表名确认是存在的，但在写入或查询时返回表名不存在，什么原因？ TDengine 中的所有名称，包括数据库名、表名等都是区分大小写的，如果这些名称在程序或 taos-CLI 中没有使用反引号（`）括起来使用，即使你输入的是大写的，引擎也会转化成小写来使用，如果名称前后加上了反引号，引擎就不会再转化成小写，会保持原样来使用。 +### 23 在 taos-CLI 中查询，字段内容不能完全显示出来怎么办？ +可以使用 \G 参数来竖式显示，如 show databases\G; （为了输入方便，在"\"后加 TAB 键，会自动补全后面的内容） +### 24 使用 taosBenchmark 测试工具写入数据查询很快，为什么我写入的数据查询非常慢？ +TDengine 在写入数据时如果有很严重的乱序写入问题，会严重影响查询性能，所以需要在写入前解决乱序的问题。如果业务是从 kafka 消费写入，请合理设计消费者，尽可能的一个子表数据由一个消费者去消费并写入，避免由设计产生的乱序。 From 9501d4f4177373d826ec967e967829789e36f05e Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 29 May 2023 19:44:11 +0800 Subject: [PATCH 147/238] feat: allow user to reset root passwd --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 3 +++ source/dnode/mnode/impl/src/mndProfile.c | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 157e37f080..6a7e36fcbb 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -83,6 +83,7 @@ extern int64_t tsVndCommitMaxIntervalMs; // mnode extern int64_t tsMndSdbWriteDelta; extern int64_t tsMndLogRetention; +extern bool tsMndSkipGrant; // monitor extern bool tsEnableMonitor; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c648f8551a..a43643a924 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -72,6 +72,7 @@ int64_t tsVndCommitMaxIntervalMs = 600 * 1000; // mnode int64_t tsMndSdbWriteDelta = 200; int64_t tsMndLogRetention = 2000; +bool tsMndSkipGrant = false; // monitor bool tsEnableMonitor = true; @@ -489,6 +490,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, 0) != 0) return -1; if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, 0) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; @@ -887,6 +889,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; + tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 01dd223b5f..a1d815189c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -245,7 +245,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0) { + if (strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1) != 0 && !tsMndSkipGrant) { mGError("user:%s, failed to login from %s since invalid pass, input:%s", pReq->info.conn.user, ip, connReq.passwd); code = TSDB_CODE_MND_AUTH_FAILURE; goto _OVER; From c1ffc64e2d79e5690f2ecb24135da4a661c7ae08 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 30 May 2023 09:54:30 +0800 Subject: [PATCH 148/238] fix: disable debug info --- source/libs/catalog/src/ctgDbg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index b90e51f1bc..7cba6ddf0d 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "trpc.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.statEnable = true}; +SCtgDebug gCTGDebug = {0}; #if 0 From 3a1fdc81d3730413c99c5cb63d5105064d2a16b8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 09:57:14 +0800 Subject: [PATCH 149/238] docs: add diff funciton to faq --- docs/zh/27-train-faq/01-faq.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index b5501d8915..15397049dd 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -258,3 +258,6 @@ TDengine 中的所有名称，包括数据库名、表名等都是区分大小 ### 24 使用 taosBenchmark 测试工具写入数据查询很快，为什么我写入的数据查询非常慢？ TDengine 在写入数据时如果有很严重的乱序写入问题，会严重影响查询性能，所以需要在写入前解决乱序的问题。如果业务是从 kafka 消费写入，请合理设计消费者，尽可能的一个子表数据由一个消费者去消费并写入，避免由设计产生的乱序。 + +### 25 我想统计下前后两条写入记录之间的时间差值是多少？ +使用 DIFF 函数，可以查看时间列或数值列前后两条记录的差值，非常方便，详细说明见 SQL手册->函数->DIFF From 88350280020e952e7c1d7a04aa044943da6355c8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 10:21:18 +0800 Subject: [PATCH 150/238] fix:deal with node is null --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c7677878bf..3752a56b92 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1160,7 +1160,7 @@ int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray **tableList pNode.tableType = TSDB_SUPER_TABLE; STableListInfo* pTableListInfo = tableListCreate(); uint8_t digest[17] = {0}; - int code = getTableList(pVnode, &pNode, pSubplan->pTagCond, pSubplan->pTagIndexCond, pTableListInfo, digest, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); + int code = getTableList(pVnode, &pNode, pSubplan ? pSubplan->pTagCond : NULL, pSubplan ? pSubplan->pTagIndexCond : NULL, pTableListInfo, digest, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); From 5d2c78ba6b5d37e3224d2d3f6587de43cdec23bd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 10:39:00 +0800 Subject: [PATCH 151/238] fix:memory leak --- source/libs/nodes/src/nodesUtilFuncs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 17a5677987..70fc45e07a 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -921,6 +921,7 @@ void nodesDestroyNode(SNode* pNode) { break; case QUERY_NODE_CREATE_TOPIC_STMT: nodesDestroyNode(((SCreateTopicStmt*)pNode)->pQuery); + nodesDestroyNode(((SCreateTopicStmt*)pNode)->pWhere); break; case QUERY_NODE_DROP_TOPIC_STMT: // no pointer field case QUERY_NODE_DROP_CGROUP_STMT: // no pointer field From f9b1906127a67711d127af40d4c4ce89964f6b97 Mon Sep 17 00:00:00 2001 From: huolibo Date: Fri, 12 May 2023 17:41:55 +0800 Subject: [PATCH 152/238] feat(driver): jdbc add tmq seek function --- .../jni/com_taosdata_jdbc_tmq_TMQConnector.h | 6 + source/client/src/clientTmqConnector.c | 118 +++++++++++++++++- 2 files changed, 120 insertions(+), 4 deletions(-) diff --git a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h index c035b6598c..094e947476 100644 --- a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h +++ b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h @@ -166,6 +166,12 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *, jobject, jlong, jlong, jobject, jobject); +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqSeekImp(JNIEnv *, jobject, jlong, jstring, jint, + jlong); + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTopicAssignmentImp(JNIEnv *, jobject, jlong, + jstring, jobject); + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientTmqConnector.c b/source/client/src/clientTmqConnector.c index 894c51d13c..5edaca48f8 100644 --- a/source/client/src/clientTmqConnector.c +++ b/source/client/src/clientTmqConnector.c @@ -17,9 +17,16 @@ #include "jniCommon.h" #include "taos.h" -int __init_tmq = 0; +int __init_tmq = 0; jmethodID g_offsetCallback; +jclass g_assignmentClass; +jmethodID g_assignmentConstructor; +jmethodID g_assignmentSetVgId; +jmethodID g_assignmentSetCurrentOffset; +jmethodID g_assignmentSetBegin; +jmethodID g_assignmentSetEnd; + void tmqGlobalMethod(JNIEnv *env) { // make sure init function executed once switch (atomic_val_compare_exchange_32(&__init_tmq, 0, 1)) { @@ -46,6 +53,38 @@ void tmqGlobalMethod(JNIEnv *env) { jniDebug("tmq method register finished"); } +int __init_assignment = 0; +void tmqAssignmentMethod(JNIEnv *env) { + // make sure init function executed once + switch (atomic_val_compare_exchange_32(&__init_assignment, 0, 1)) { + case 0: + break; + case 1: + do { + taosMsleep(0); + } while (atomic_load_32(&__init_assignment) == 1); + case 2: + return; + } + + if (g_vm == NULL) { + (*env)->GetJavaVM(env, &g_vm); + } + + jclass assignment = (*env)->FindClass(env, "com/taosdata/jdbc/tmq/Assignment"); + g_assignmentClass = (*env)->NewGlobalRef(env, assignment); + g_assignmentConstructor = (*env)->GetMethodID(env, g_assignmentClass, "", "()V"); + g_assignmentSetVgId = (*env)->GetMethodID(env, g_assignmentClass, "setVgId", "(I)V"); // int + g_assignmentSetCurrentOffset = (*env)->GetMethodID(env, g_assignmentClass, "setCurrentOffset", "(J)V"); // long + g_assignmentSetBegin = (*env)->GetMethodID(env, g_assignmentClass, "setBegin", "(J)V"); // long + g_assignmentSetEnd = (*env)->GetMethodID(env, g_assignmentClass, "setEnd", "(J)V"); // long + + (*env)->DeleteLocalRef(env, assignment); + + atomic_store_32(&__init_assignment, 2); + jniDebug("tmq method assignment finished"); +} + // deprecated void commit_cb(tmq_t *tmq, int32_t code, void *param) { JNIEnv *env = NULL; @@ -266,8 +305,9 @@ JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqCommitAsync(JN tmq_commit_async(tmq, res, commit_cb, consumer); } -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_consumerCommitAsync(JNIEnv *env, jobject jobj, jlong jtmq, - jlong jres, jobject offset) { +JNIEXPORT void JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_consumerCommitAsync(JNIEnv *env, jobject jobj, + jlong jtmq, jlong jres, + jobject offset) { tmqGlobalMethod(env); tmq_t *tmq = (tmq_t *)jtmq; if (tmq == NULL) { @@ -369,7 +409,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res); return JNI_FETCH_END; } else { - jniError("jobj:%p, conn:%p, query interrupted, tmq fetch block error code:%d, msg:%s", jobj, tscon, error_code, taos_errstr(tres)); + jniError("jobj:%p, conn:%p, query interrupted, tmq fetch block error code:%d, msg:%s", jobj, tscon, error_code, + taos_errstr(tres)); return JNI_RESULT_SET_NULL; } } @@ -399,3 +440,72 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( (*env)->CallVoidMethod(env, rowobj, g_blockdataSetByteArrayFp, jniFromNCharToByteArray(env, (char *)data, len)); return JNI_SUCCESS; } + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqSeekImp(JNIEnv *env, jobject jobj, jlong jtmq, + jstring jtopic, jint partition, + jlong offset) { + tmq_t *tmq = (tmq_t *)jtmq; + if (tmq == NULL) { + jniDebug("jobj:%p, tmq is closed", jobj); + return TMQ_CONSUMER_NULL; + } + + if (jtopic == NULL) { + jniDebug("jobj:%p, topic is null", jobj); + return TMQ_TOPIC_NULL; + } + const char *topicName = (*env)->GetStringUTFChars(env, jtopic, NULL); + + int32_t res = tmq_offset_seek(tmq, topicName, partition, offset); + + if (res != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, tmq seek error, code:%d, msg:%s", jobj, res, tmq_err2str(res)); + } + + (*env)->ReleaseStringUTFChars(env, jtopic, topicName); + return (jint)res; +} + +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTopicAssignmentImp(JNIEnv *env, jobject jobj, + jlong jtmq, jstring jtopic, + jobject jarrayList) { + tmqAssignmentMethod(env); + tmq_t *tmq = (tmq_t *)jtmq; + if (tmq == NULL) { + jniDebug("jobj:%p, tmq is closed", jobj); + return TMQ_CONSUMER_NULL; + } + + if (jtopic == NULL) { + jniDebug("jobj:%p, topic is null", jobj); + return TMQ_TOPIC_NULL; + } + + const char *topicName = (*env)->GetStringUTFChars(env, jtopic, NULL); + + tmq_topic_assignment *pAssign = NULL; + int32_t numOfAssignment = 0; + int32_t res = tmq_get_topic_assignment(tmq, topicName, &pAssign, &numOfAssignment); + + if (res != TSDB_CODE_SUCCESS) { + (*env)->ReleaseStringUTFChars(env, jtopic, topicName); + jniError("jobj:%p, tmq get topic assignment error, topic:%s, code:%d, msg:%s", jobj, topicName, res, + tmq_err2str(res)); + taosMemoryFree(pAssign); + return (jint)res; + } + + (*env)->ReleaseStringUTFChars(env, jtopic, topicName); + + for (int i = 0; i < numOfAssignment; ++i) { + tmq_topic_assignment assignment = pAssign[i]; + jobject jassignment = (*env)->NewObject(env, g_assignmentClass, g_assignmentConstructor); + (*env)->CallVoidMethod(env, jassignment, g_assignmentSetVgId, assignment.vgId); + (*env)->CallVoidMethod(env, jassignment, g_assignmentSetCurrentOffset, assignment.currentOffset); + (*env)->CallVoidMethod(env, jassignment, g_assignmentSetBegin, assignment.begin); + (*env)->CallVoidMethod(env, jassignment, g_assignmentSetEnd, assignment.end); + (*env)->CallBooleanMethod(env, jarrayList, g_arrayListAddFp, jassignment); + } + taosMemoryFree(pAssign); + return JNI_SUCCESS; +} From 79031c47a2492cdc445c01cb49f3ab11a8360ccc Mon Sep 17 00:00:00 2001 From: huolibo Date: Thu, 25 May 2023 17:39:01 +0800 Subject: [PATCH 153/238] enh: add get offset --- .../client/jni/com_taosdata_jdbc_tmq_TMQConnector.h | 7 +++++++ source/client/src/clientTmqConnector.c | 13 +++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h index 094e947476..422bcd57ac 100644 --- a/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h +++ b/source/client/jni/com_taosdata_jdbc_tmq_TMQConnector.h @@ -158,6 +158,13 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetVgroupId(JN */ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableName(JNIEnv *, jobject, jlong); +/* + * Class: com_taosdata_jdbc_tmq_TMQConnector + * Method: tmqGetOffset + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetOffset(JNIEnv *, jobject, jlong); + /* * Class: com_taosdata_jdbc_tmq_TMQConnector * Method: fetchBlockImp diff --git a/source/client/src/clientTmqConnector.c b/source/client/src/clientTmqConnector.c index 5edaca48f8..6b0bfc09e8 100644 --- a/source/client/src/clientTmqConnector.c +++ b/source/client/src/clientTmqConnector.c @@ -390,6 +390,15 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTableNam return (*env)->NewStringUTF(env, tmq_get_table_name(res)); } +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetOffset(JNIEnv *env, jobject jobj, jlong jres) { + TAOS_RES *res = (TAOS_RES *)jres; + if (res == NULL) { + jniDebug("jobj:%p, invalid res handle", jobj); + return NULL; + } + return tmq_get_vgroup_offset(res); +} + JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(JNIEnv *env, jobject jobj, jlong con, jlong res, jobject rowobj, jobject arrayListObj) { @@ -491,7 +500,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTopicAssign (*env)->ReleaseStringUTFChars(env, jtopic, topicName); jniError("jobj:%p, tmq get topic assignment error, topic:%s, code:%d, msg:%s", jobj, topicName, res, tmq_err2str(res)); - taosMemoryFree(pAssign); + tmq_free_assignment(pAssign); return (jint)res; } @@ -506,6 +515,6 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetTopicAssign (*env)->CallVoidMethod(env, jassignment, g_assignmentSetEnd, assignment.end); (*env)->CallBooleanMethod(env, jarrayList, g_arrayListAddFp, jassignment); } - taosMemoryFree(pAssign); + tmq_free_assignment(pAssign); return JNI_SUCCESS; } From a70cceb440f431bcb6c96fdb41bd4b21e22d7aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E8=93=93?= Date: Fri, 26 May 2023 11:14:30 +0800 Subject: [PATCH 154/238] fix: add return value --- source/client/src/clientTmqConnector.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientTmqConnector.c b/source/client/src/clientTmqConnector.c index 6b0bfc09e8..08c802d967 100644 --- a/source/client/src/clientTmqConnector.c +++ b/source/client/src/clientTmqConnector.c @@ -394,7 +394,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetOffset(JNI TAOS_RES *res = (TAOS_RES *)jres; if (res == NULL) { jniDebug("jobj:%p, invalid res handle", jobj); - return NULL; + return -1; } return tmq_get_vgroup_offset(res); } From 2e5ac6b0c2007ec488cf6efde8f8c864fd6f1209 Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 30 May 2023 11:33:46 +0800 Subject: [PATCH 155/238] fix: change return value --- source/client/src/clientTmqConnector.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmqConnector.c b/source/client/src/clientTmqConnector.c index 08c802d967..6ec82aa6ef 100644 --- a/source/client/src/clientTmqConnector.c +++ b/source/client/src/clientTmqConnector.c @@ -375,7 +375,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetVgroupId(JN TAOS_RES *res = (TAOS_RES *)jres; if (res == NULL) { jniDebug("jobj:%p, invalid res handle", jobj); - return -1; + return JNI_RESULT_SET_NULL; } return tmq_get_vgroup_id(res); } @@ -394,7 +394,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqGetOffset(JNI TAOS_RES *res = (TAOS_RES *)jres; if (res == NULL) { jniDebug("jobj:%p, invalid res handle", jobj); - return -1; + return JNI_RESULT_SET_NULL; } return tmq_get_vgroup_offset(res); } From 0d2fa6f1e648a083af30b2fa1d0f5d8c95117421 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 30 May 2023 11:49:00 +0800 Subject: [PATCH 156/238] docs: refine python connector doc for req_id (#21526) * docs: update csharp connector status * docs: fix csharp ws bulk pulling * docs: clarify database param is optional to websocket dsn * docs: fix java connector mistake * fix: a few typos * fix: many typos * docs: java connector support subscription over websocket * fix: update 3.0 connector feature matrix * docs: fix typos and use 3.1.0 in jdbc driver * fix: update matrix for schemaless java status * docs: use latest jdbc version in connect section * docs: refine python connector doc for req_id --- .../en/14-reference/03-connector/07-python.mdx | 18 ++++++------------ docs/zh/08-connector/30-python.mdx | 18 +++++++----------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index a83f2047d0..b263af8ea6 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -362,7 +362,7 @@ By using the optional req_id parameter, you can specify a request ID that can be ##### TaosConnection class -The `TaosConnection` class contains both an implementation of the PEP249 Connection interface (e.g., the `cursor()` method and the `close()` method) and many extensions (e.g., the `execute()`, `query()`, `schemaless_insert()`, and `subscribe()` methods). +As the way to connect introduced above but add `req_id` argument. ```python title="execute method" {{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:insert}} @@ -372,13 +372,9 @@ The `TaosConnection` class contains both an implementation of the PEP249 Connect {{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:query}} ``` -:::tip -The queried results can only be fetched once. For example, only one of `fetch_all()` and `fetch_all_into_dict()` can be used in the example above. Repeated fetches will result in an empty list. -::: - ##### Use of TaosResult class -In the above example of using the `TaosConnection` class, we have shown two ways to get the result of a query: `fetch_all()` and `fetch_all_into_dict()`. In addition, `TaosResult` also provides methods to iterate through the result set by rows (`rows_iter`) or by data blocks (`blocks_iter`). Using these two methods will be more efficient in scenarios where the query has a large amount of data. +As the way to fetch data introduced above but add `req_id` argument. ```python title="blocks_iter method" {{#include docs/examples/python/result_set_with_req_id_examples.py}} @@ -391,17 +387,12 @@ The `TaosConnection` class and the `TaosResult` class already implement all the {{#include docs/examples/python/cursor_usage_native_reference_with_req_id.py}} ``` -:::note -The TaosCursor class uses native connections for write and query operations. In a client-side multi-threaded scenario, this cursor instance must remain thread exclusive and cannot be shared across threads for use, otherwise, it will result in errors in the returned results. - -::: - ##### Use of TaosRestCursor class -The `TaosRestCursor` class is an implementation of the PEP249 Cursor interface. +As the way to connect introduced above but add `req_id` argument. ```python title="Use of TaosRestCursor" {{#include docs/examples/python/connect_rest_with_req_id_examples.py:basic}} @@ -421,8 +412,11 @@ The `RestClient` class is a direct wrapper for the [REST API](/reference/rest-ap For a more detailed description of the `sql()` method, please refer to [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html). + +As the way to connect introduced above but add `req_id` argument. + ```python {{#include docs/examples/python/connect_websocket_with_req_id_examples.py:basic}} ``` diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index 1cff142e11..1037d66f17 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -362,7 +362,7 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 ##### TaosConnection 类的使用 -`TaosConnection` 类既包含对 PEP249 Connection 接口的实现(如：`cursor`方法和 `close` 方法)，也包含很多扩展功能（如： `execute`、 `query`、`schemaless_insert` 和 `subscribe` 方法。 +类似上文介绍的使用方法，增加 `req_id` 参数。 ```python title="execute 方法" {{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:insert}} @@ -372,13 +372,9 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 {{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:query}} ``` -:::tip -查询结果只能获取一次。比如上面的示例中 `fetch_all()` 和 `fetch_all_into_dict()` 只能用一个。重复获取得到的结果为空列表。 -::: - ##### TaosResult 类的使用 -上面 `TaosConnection` 类的使用示例中，我们已经展示了两种获取查询结果的方法： `fetch_all()` 和 `fetch_all_into_dict()`。除此之外 `TaosResult` 还提供了按行迭代(`rows_iter`)或按数据块迭代(`blocks_iter`)结果集的方法。在查询数据量较大的场景，使用这两个方法会更高效。 +类似上文介绍的使用方法，增加 `req_id` 参数。 ```python title="blocks_iter 方法" {{#include docs/examples/python/result_set_with_req_id_examples.py}} @@ -391,14 +387,11 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 {{#include docs/examples/python/cursor_usage_native_reference_with_req_id.py}} ``` -:::note -TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线程的场景下，这个游标实例必须保持线程独享，不能跨线程共享使用，否则会导致返回结果出现错误。 - -::: - +类似上文介绍的使用方法，增加 `req_id` 参数。 + ##### TaosRestCursor 类的使用 `TaosRestCursor` 类是对 PEP249 Cursor 接口的实现。 @@ -420,8 +413,11 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 对于 `sql()` 方法更详细的介绍， 请参考 [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html)。 + +类似上文介绍的使用方法，增加 `req_id` 参数。 + ```python {{#include docs/examples/python/connect_websocket_with_req_id_examples.py:basic}} ``` From 757ac9b1d9749be7a2d52de06bcc67e4b7af8419 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 14:20:21 +0800 Subject: [PATCH 157/238] fix:add test cases --- tests/system-test/7-tmq/stbFilterWhere.py | 72 ++++++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/tests/system-test/7-tmq/stbFilterWhere.py b/tests/system-test/7-tmq/stbFilterWhere.py index baf042d71c..8d8d046cef 100644 --- a/tests/system-test/7-tmq/stbFilterWhere.py +++ b/tests/system-test/7-tmq/stbFilterWhere.py @@ -56,7 +56,7 @@ class TDTestCase: startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) return - def tmqCaseError(self, topicName, condition): + def tmqCase_columnError(self, topicName, condition): tdLog.printNoPrefix("======== test case error: ") paraDict = {'dbName': 'dbt', 'dropFlag': 1, @@ -140,14 +140,82 @@ class TDTestCase: tdLog.printNoPrefix("======== test case end ...... ") + def tmqCase_addNewTable_dropTag(self, topicName, condition): + tdLog.printNoPrefix("======== test case1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 2, + 'showMsg': 1, + 'showRow': 1} + + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with tag filter") + topicString = "create topic %s as stable %s.%s where %s" %(topicName, paraDict['dbName'], paraDict['stbName'], condition) + tdLog.info("create topic sql: %s"%topicString) + tdSql.execute(topicString) + + queryString = "select * from %s.%s where %s" %(paraDict['dbName'], paraDict['stbName'], condition) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows() + 1) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicName + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + #add new table with one data + tdLog.info("start insert data") + insertString = "insert into %s.tmp using %s.%s tags(1, 1, 1, 't4', 't5') values(now, 1, 1, 1, 'c4', 'c5', now)" %(paraDict['dbName'], paraDict['dbName'], paraDict['stbName']) + tdSql.execute(insertString) + + #test drop tag + tdSql.error("alter stable %s.%s drop tag t1" %(paraDict['dbName'], paraDict['stbName'])) + tdSql.execute("alter stable %s.%s drop tag t2" %(paraDict['dbName'], paraDict['stbName'])) + tdSql.execute("alter stable %s.%s drop column c2" %(paraDict['dbName'], paraDict['stbName'])) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + tdLog.printNoPrefix("======== test case1 end ...... ") + def run(self): tdSql.prepare() self.prepareTestEnv() - self.tmqCaseError("t1", "c1 = 4 and t1 = 3") + self.tmqCase_columnError("t1", "c1 = 4 and t1 = 3") self.tmqCase("t2", "2 > 1") self.tmqCase("t3", "t4 = 'beijing'") self.tmqCase("t4", "t4 > t3") self.tmqCase("t5", "t3 = t4") + self.tmqCase_addNewTable_dropTag("t6", "t1 = 1") def stop(self): tdSql.close() From 6a7d8858dc516c3b1fe79ea2194e5159871ac99e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 14:36:51 +0800 Subject: [PATCH 158/238] test: add smaBasic.py test case --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/smaBasic.py | 297 ++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 tests/system-test/2-query/smaBasic.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcfb392bb4..f0543f2456 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -294,6 +294,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaBasic.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py new file mode 100644 index 0000000000..917a2d3a08 --- /dev/null +++ b/tests/system-test/2-query/smaBasic.py @@ -0,0 +1,297 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + self.ts += 1 + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null)" + else: + value += f"{c2})" + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1600000000000 + self.childCnt = 5 + self.childRow = 2000000 + self.batchSize = 2000 + self.c1_values = [] + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 5 replica 3" + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # insert data + self.insertData() + + # query sql value + def queryValue(self, sql): + tdSql.query(sql) + return tdSql.getData(0, 0) + + # sum + def checkCorrentSum(self): + # query count + sql = "select sum(c1) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c1Cnt: + tdLog.exit(f"Sum Not Expect. expect={self.c1Cnt} query={val} sql:{sql}") + return + + # not + sql1 = "select sum(c1) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + # over than + sql1 = "select sum(c1) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + tdLog.info(f"check correct sum on c1 successfully.") + + # check result + def checkResult(self, fun, val, val1, val2, sql1, sql2): + if fun == "count": + if val != val1 + val2: + tdLog.exit(f"{fun} NOT SAME. val != val1 + val2. val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "max": + if val != max([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != max(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "min": + if val != min([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != min(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + + # sum + def checkCorrentFun(self, fun, expectVal): + # query + sql = f"select {fun}(c2) from st" + val = self.queryValue(sql) + if val != expectVal: + tdLog.exit(f"{fun} Not Expect. expect={expectVal} query={val} sql:{sql}") + return + + # not + sql1 = f"select {fun}(c2) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # over than + sql1 = f"select {fun}(c2) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # successful + tdLog.info(f"check correct {fun} on c2 successfully.") + + # check query corrent + def checkCorrect(self): + # count + self.checkCorrentFun("count", self.c2Cnt) + # max + self.checkCorrentFun("max", self.c2Max) + # min + self.checkCorrentFun("min", self.c2Min) + # sum + self.checkCorrentSum() + + # c2 sum + sql = "select sum(c2) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c2Sum: + tdLog.exit(f"c2 Sum Not Expect. expect={self.c2Sum} query={val} sql:{sql}") + return + + def checkPerformance(self): + # have sma caculate + sql1 = "select count(*) from st" + stime = time.time() + tdSql.execute(sql1, 1) + spend1 = time.time() - stime + + + # no sma caculate + sql2 = "select count(*) from st where c2 != 8764231 or c2 is null" + stime = time.time() + tdSql.execute(sql2, 1) + spend2 = time.time() - stime + + time1 = "%.2f"%(spend1*1000) + time2 = "%.2f"%(spend2*1000) + if spend2 < spend1 * 5: + tdLog.exit(f"performance not passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + return + tdLog.info(f"performance passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # query + self.checkCorrect() + + # performance + self.checkPerformance() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From ed9eaf541d1d95afac64c18898d2f384a0308031 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 14:37:46 +0800 Subject: [PATCH 159/238] test: add smaBasic.py test case --- tests/system-test/2-query/smaBasic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py index 917a2d3a08..26adc03d2b 100644 --- a/tests/system-test/2-query/smaBasic.py +++ b/tests/system-test/2-query/smaBasic.py @@ -117,7 +117,7 @@ class TDTestCase: self.ts = 1600000000000 self.childCnt = 5 self.childRow = 2000000 - self.batchSize = 2000 + self.batchSize = 5000 self.c1_values = [] # total From 7d699ae579088ce2f075417a6925a0a3eae260e8 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Tue, 30 May 2023 15:16:11 +0800 Subject: [PATCH 160/238] fix sliding window issue --- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 11 ++++++----- source/libs/stream/src/streamState.c | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2d2f377041..f11b083ad6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1883,7 +1883,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pInfo->pRecoverRes != NULL) { pInfo->blockRecoverContiCnt++; calBlockTbName(pInfo, pInfo->pRecoverRes); - if (pInfo->pUpdateInfo) { + if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) { if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) { TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index aa0aa9799b..2cdf9bb15c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1623,7 +1623,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; - if (!pScanInfo->igCheckUpdate && !pScanInfo->pUpdateInfo) { + if (!pScanInfo->pUpdateInfo) { pScanInfo->pUpdateInfo = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark); } @@ -2150,28 +2150,29 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB } void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SInterval* pInterval) { - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); TSKEY* tsData = (TSKEY*)pStartCol->pData; - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); TSKEY* tsEndData = (TSKEY*)pEndCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; int32_t chId = getChildIndex(pBlock); for (int32_t i = 0; i < pBlock->info.rows; i++) { TSKEY winTs = tsData[i]; - while (winTs < tsEndData[i]) { + while (winTs <= tsEndData[i]) { SWinKey winRes = {.ts = winTs, .groupId = groupIdData[i]}; void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey)); if (chIds) { SArray* chArray = *(SArray**)chIds; int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); if (index != -1) { - qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId); + qDebug("===stream===retrive window %" PRId64 " delete child id %d", winRes.ts, chId); taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over taosArrayDestroy(chArray); taosHashRemove(pMap, &winRes, sizeof(SWinKey)); + qDebug("===stream===retrive pull data over.window %" PRId64 , winRes.ts); } } } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ba24a581e9..71a21ac150 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -1062,7 +1062,7 @@ _end: } int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { - qWarn("try to write to cf parname"); + qDebug("try to write to cf parname"); #ifdef USE_ROCKSDB if (tSimpleHashGetSize(pState->parNameMap) > MAX_TABLE_NAME_NUM) { if (tSimpleHashGet(pState->parNameMap, &groupId, sizeof(int64_t)) == NULL) { From de7231e59e839c8d6fc58439bc0bdf2b2c805e75 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 15:35:59 +0800 Subject: [PATCH 161/238] fix(stream): fix memory leak in handling dispatch msg when output buffer is full. --- source/libs/stream/src/stream.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 8cc1ef1dd3..d3e4b23ad1 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -129,11 +129,11 @@ int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pR qDebug("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId, pTask->id.idStr); } else { - if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock) == 0) { - status = TASK_INPUT_STATUS__NORMAL; - } else { // input queue is full, upstream is blocked now - status = TASK_INPUT_STATUS__BLOCKED; - } + int32_t code = tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock); + // input queue is full, upstream is blocked now + status = (code == TSDB_CODE_SUCCESS)? TASK_INPUT_STATUS__NORMAL:TASK_INPUT_STATUS__BLOCKED; + + } // rsp by input status @@ -303,6 +303,7 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { taosFreeQitem(pItem); return -1; } + taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { @@ -310,6 +311,8 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); + destroyStreamDataBlock((SStreamDataBlock*) pItem); + taosFreeQitem(pItem); return -1; } From 772ef3bd19b5264d54766bb7397b7c135d741cde Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 15:39:15 +0800 Subject: [PATCH 162/238] test: modfiy insert rows to small --- tests/system-test/2-query/smaBasic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py index 26adc03d2b..43c379ee53 100644 --- a/tests/system-test/2-query/smaBasic.py +++ b/tests/system-test/2-query/smaBasic.py @@ -116,9 +116,8 @@ class TDTestCase: # init self.ts = 1600000000000 self.childCnt = 5 - self.childRow = 2000000 + self.childRow = 1000000 self.batchSize = 5000 - self.c1_values = [] # total self.c1Cnt = 0 @@ -262,7 +261,7 @@ class TDTestCase: time1 = "%.2f"%(spend1*1000) time2 = "%.2f"%(spend2*1000) - if spend2 < spend1 * 5: + if spend2 < spend1 * 8: tdLog.exit(f"performance not passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") return tdLog.info(f"performance passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") From b8a4df414699a94e0c4a2c775d101ae8df3c6123 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 16:50:20 +0800 Subject: [PATCH 163/238] fix:avoid put tag to select if where condition is always true or false --- source/libs/parser/src/parTranslater.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 102b7479be..c10ee5d988 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5899,15 +5899,15 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* nodesDestroyList(colCxt.pTags); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Columns are forbidden in where clause"); } - if (NULL == colCxt.pTags) { - for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) { - SSchema* tag = &pMeta->schema[pMeta->tableInfo.numOfColumns + i]; + if (NULL == colCxt.pTags) { // put one column to select +// for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { + SSchema* column = &pMeta->schema[0]; SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - strcpy(col->colName, tag->name); + strcpy(col->colName, column->name); strcpy(col->node.aliasName, col->colName); strcpy(col->node.userAlias, col->colName); addTagList(&colCxt.pTags, (SNode*)col); - } +// } } *ppProjection = colCxt.pTags; From fcdf25793b5890b7f39d2f47ab1b004a43194877 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 30 May 2023 16:52:53 +0800 Subject: [PATCH 164/238] fix:remove useless code --- tests/script/tsim/tmq/topic.sim | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index 85a23055a3..073c8c2532 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -108,7 +108,6 @@ if $rows != 6 then return -1 endi -return -1 sql create topic topic_stable_1 as stable stb where t1 > 0 sql create topic topic_stable_1 as stable stb where t1 > 0 and t1 < 0 sql create topic topic_stable_1 as stable stb where 1 > 0 From e96ed81ec0df8c6eacdaa0c38c44e8099c9c3346 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 17:29:01 +0800 Subject: [PATCH 165/238] fix(stream): add input queue capacity check. --- source/libs/stream/src/stream.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index d3e4b23ad1..0d772247b4 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -132,8 +132,6 @@ int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pR int32_t code = tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock); // input queue is full, upstream is blocked now status = (code == TSDB_CODE_SUCCESS)? TASK_INPUT_STATUS__NORMAL:TASK_INPUT_STATUS__BLOCKED; - - } // rsp by input status @@ -304,10 +302,15 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { return -1; } - taosWriteQitem(pTask->inputQueue->queue, pItem); + int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); + if (code != TSDB_CODE_SUCCESS) { + streamDataSubmitDestroy(px); + taosFreeQitem(pItem); + return code; + } } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { - if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && (tInputQueueIsFull(pTask))) { + if (/*(pTask->taskLevel == TASK_LEVEL__SOURCE) && */(tInputQueueIsFull(pTask))) { qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); @@ -317,10 +320,16 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { } qDebug("s-task:%s data block enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size); - taosWriteQitem(pTask->inputQueue->queue, pItem); + int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamDataBlock((SStreamDataBlock*) pItem); + taosFreeQitem(pItem); + return code; + } } else if (type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (type == STREAM_INPUT__GET_RES) { + // use the default memory limit, refactor later. taosWriteQitem(pTask->inputQueue->queue, pItem); qDebug("s-task:%s data res enqueue, current(blocks:%d, size:%.2fMiB)", pTask->id.idStr, total, size); } From 937f67f07b0a89dc1df5a5e01c4dbfa2e31b76e5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 18:02:42 +0800 Subject: [PATCH 166/238] fix(stream): fix invalid free. --- source/libs/stream/src/stream.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 0d772247b4..03013dbf3e 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -315,7 +315,6 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); destroyStreamDataBlock((SStreamDataBlock*) pItem); - taosFreeQitem(pItem); return -1; } From d9cc9f21574f4799e827c5792e85e4efa4ce3db1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 18:03:21 +0800 Subject: [PATCH 167/238] fix(stream): remove invalid free. --- source/libs/stream/src/stream.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 03013dbf3e..6b69f19427 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -322,7 +322,6 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { int32_t code = taosWriteQitem(pTask->inputQueue->queue, pItem); if (code != TSDB_CODE_SUCCESS) { destroyStreamDataBlock((SStreamDataBlock*) pItem); - taosFreeQitem(pItem); return code; } } else if (type == STREAM_INPUT__CHECKPOINT) { From fc20ca8423dbe4b637d3465adea0f8fce692072e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 30 May 2023 18:42:23 +0800 Subject: [PATCH 168/238] fix(stream): set the correct rps flag as the dispatch rsp. --- source/libs/stream/src/stream.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 6b69f19427..8bca0e6ec2 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -233,8 +233,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { - ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED); - qDebug("s-task:%s receive dispatch rsp, code: %x", pTask->id.idStr, code); + qDebug("s-task:%s receive dispatch rsp, status:%d code:%d", pTask->id.idStr, pRsp->inputStatus, code); if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); @@ -246,13 +245,16 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); ASSERT(old == TASK_OUTPUT_STATUS__WAIT); + + // the input queue of the (down stream) task that receive the output data is full, so the TASK_INPUT_STATUS_BLOCKED is rsp + // todo we need to send EMPTY PACKAGE to detect if the input queue is available for output of upstream task, every 50 ms. if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { // TODO: init recover timer - ASSERT(0); + qError("s-task:%s inputQ of downstream task:0x%x is full, need to block output", pTask->id.idStr, pRsp->downstreamTaskId); return 0; } - // continue dispatch one block to down stream in pipeline + // otherwise, continue dispatch the first block to down stream task in pipeline streamDispatchStreamBlock(pTask); return 0; } From 67becbd4d1562dba82fcbb7b31a16bd7a0c4446a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 31 May 2023 01:10:36 +0000 Subject: [PATCH 169/238] change db param --- source/libs/stream/src/streamBackendRocksdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f7638a42ae..f7a8ec4564 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -92,6 +92,7 @@ void* streamBackendInit(const char* path) { rocksdb_options_set_recycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 2); rocksdb_options_set_info_log_level(opts, 0); + rocksdb_options_set_db_write_buffer_size(opts, 256 << 20); pHandle->env = env; pHandle->dbOpt = opts; From 23bcfc2fd0714f9ef549c638635fc26ec7e31f2d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 31 May 2023 09:16:47 +0800 Subject: [PATCH 170/238] enh: stmt column length validation --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 5 +- source/libs/parser/src/parInsertStmt.c | 7 +- tests/script/api/batchprepare.c | 93 +++++++++++++++++++------- 4 files changed, 79 insertions(+), 28 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 8be5cb4d41..e04bdd1b07 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -145,7 +145,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); // for stmt bind -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen); void tColDataSortMerge(SArray *colDataArr); // for raw block diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 55204045ba..688f3b006c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2503,7 +2503,7 @@ _exit: return code; } -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen) { int32_t code = 0; if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { @@ -2515,6 +2515,9 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { if (pBind->is_null && pBind->is_null[i]) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; + } else if (pBind->length[i] > buffMaxLen) { + uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen); + return TSDB_CODE_INVALID_PARA; } else { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 922a0f45ff..8284913975 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -266,7 +266,10 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in pBind = bind + c; } - tColDataAddValueByBind(pCol, pBind); + code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); + if (code) { + goto _return; + } } qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); @@ -309,7 +312,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu pBind = bind; } - tColDataAddValueByBind(pCol, pBind); + tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); qDebug("stmt col %d bind %d rows data", colIdx, rowNum); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 99507ef5c3..80bf5b90af 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -16,8 +16,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; -int32_t optrIdxList[] = {0, 7}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR}; +int32_t optrIdxList[] = {5, 11}; typedef struct { char* oper; @@ -123,6 +123,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos); +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos); enum { TTYPE_INSERT = 1, @@ -190,6 +191,7 @@ CaseCfg gCase[] = { {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:NG-TBNEXISTS",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, false, insertNonExistsTb, 10, 10, 1, 3, 0, 0, 1, -1}, + {"query:NG-VARLENERR",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, true, insertVarLenErr, 10, 10, 1, 3, 0, 0, 1, -1}, // {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, // {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, @@ -319,7 +321,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper #if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper - .bindNullNum = 1, + .bindNullNum = 0, .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, @@ -329,18 +331,19 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindTagNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, - .bindColTypeList = NULL, + .bindColTypeList = bindColTypeList, .optrIdxListNum = 0, - .optrIdxList = NULL, + .optrIdxList = optrIdxList, .checkParamNum = false, .printRes = true, .runTimes = 0, .caseRunIdx = -1, - //.optrIdxListNum = tListLen(optrIdxList), - //.optrIdxList = optrIdxList, - //.bindColTypeNum = tListLen(bindColTypeList), - //.bindColTypeList = bindColTypeList, - .caseIdx = 8, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseRunIdx = -1, + .caseIdx = 24, .caseNum = 1, .caseRunNum = 1, }; @@ -1439,14 +1442,17 @@ void bpShowBindParam(TAOS_MULTI_BIND *bind, int32_t num) { } } -int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { +int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, bool expectFail) { static int32_t n = 0; - bpCheckColFields(stmt, bind); + if (!expectFail) { + bpCheckColFields(stmt, bind); + } if (gCurCase->bindRowNum > 1) { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1454,6 +1460,7 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { if (taos_stmt_bind_single_param_batch(stmt, bind+i, i)) { + if (expectFail) continue; printf("!!!taos_stmt_bind_single_param_batch %d error:%s\n", taos_stmt_errstr(stmt), i); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1463,12 +1470,14 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); } } else { if (taos_stmt_bind_param(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1531,7 +1540,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1583,7 +1592,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1641,7 +1650,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1691,7 +1700,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1759,7 +1768,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1811,7 +1820,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1872,7 +1881,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1938,7 +1947,7 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2005,7 +2014,7 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos) { if (gCaseCtrl.checkParamNum) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2065,7 +2074,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2119,7 +2128,7 @@ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2167,7 +2176,7 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2234,6 +2243,42 @@ int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) { return 0; } +void bpAddWrongVarBuffLen(TAOS_MULTI_BIND* pBind) { + for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { + if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) { + *pBind[i].length += 100; + } + } +} + +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + code = bpSetTableNameTags(&data, 0, "t0", stmt); + if (code != 0){ + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpAddWrongVarBuffLen(data.pBind); + + if (bpBindParam(stmt, data.pBind, true)) { + exit(1); + } + + destroyData(&data); + + return 0; +} int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { From e4866d1803c6987a377ab3eb8dfd96a9cdbe2422 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 31 May 2023 10:38:22 +0800 Subject: [PATCH 171/238] fix:ci test cases error --- tests/script/tsim/tmq/topic.sim | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index 073c8c2532..78c4c561af 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -109,12 +109,14 @@ if $rows != 6 then endi sql create topic topic_stable_1 as stable stb where t1 > 0 -sql create topic topic_stable_1 as stable stb where t1 > 0 and t1 < 0 -sql create topic topic_stable_1 as stable stb where 1 > 0 -sql create topic topic_stable_1 as stable stb where last(t1) > 0 -sql create topic topic_stable_1 as stable stb where tbname is not null -sql create topic topic_stable_1 as stable stb where tbname > 'a' -sql create topic topic_stable_1 as stable stb where tbname > 0 and xx < 0 -sql create topic topic_stable_1 as stable stb where tbname > 0 and c1 < 0 +sql create topic topic_stable_2 as stable stb where t1 > 0 and t1 < 0 +sql create topic topic_stable_3 as stable stb where 1 > 0 +sql create topic topic_stable_4 as stable stb where abs(t1) > 0 +sql_error create topic topic_stable_5 as stable stb where last(t1) > 0 +sql_error create topic topic_stable_5 as stable stb where sum(t1) > 0 +sql create topic topic_stable_6 as stable stb where tbname is not null +sql create topic topic_stable_7 as stable stb where tbname > 'a' +sql_error create topic topic_stable_8 as stable stb where tbname > 0 and xx < 0 +sql_error create topic topic_stable_9 as stable stb where tbname > 0 and c1 < 0 system sh/exec.sh -n dnode1 -s stop -x SIGINT From 3b0a6d7f0b0e5082ef074c536b0afd45dff3c36a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 31 May 2023 03:15:25 +0000 Subject: [PATCH 172/238] change db param --- include/common/tglobal.h | 2 +- source/common/src/tglobal.c | 16 ++++++++++------ source/libs/stream/src/streamBackendRocksdb.c | 12 +++++++++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 157e37f080..ced94e0f11 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -29,7 +29,6 @@ extern "C" { #define SLOW_LOG_TYPE_OTHERS 0x4 #define SLOW_LOG_TYPE_ALL 0xFFFFFFFF - // cluster extern char tsFirst[]; extern char tsSecond[]; @@ -181,6 +180,7 @@ extern bool tsDisableStream; extern int64_t tsStreamBufferSize; extern int64_t tsCheckpointInterval; extern bool tsFilterScalarMode; +extern int32_t tsMaxStreamBackendCache; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c648f8551a..cc368167f8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -60,6 +60,7 @@ int32_t tsNumOfQnodeQueryThreads = 4; int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeStreamThreads = 4; int32_t tsNumOfSnodeWriteThreads = 1; +int32_t tsMaxStreamBackendCache = 256; // M // sync raft int32_t tsElectInterval = 25 * 1000; @@ -105,7 +106,7 @@ int32_t tsQueryPolicy = 1; int32_t tsQueryRspPolicy = 0; int64_t tsQueryMaxConcurrentTables = 200; // unit is TSDB_TABLE_NUM_UNIT bool tsEnableQueryHb = false; -bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true +bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -117,8 +118,8 @@ int32_t tsRedirectFactor = 2; int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; -int32_t tsMetaCacheMaxSize = -1; // MB -int32_t tsSlowLogThreshold = 3; // seconds +int32_t tsMetaCacheMaxSize = -1; // MB +int32_t tsSlowLogThreshold = 3; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; /* @@ -349,7 +350,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; - if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) + return -1; if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, 1) != 0) return -1; if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, true) != 0) return -1; if (cfgAddString(pCfg, "slowLogScope", "", true) != 0) return -1; @@ -524,6 +526,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1; if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, 0) != 0) return -1; GRANT_CFG_ADD; return 0; @@ -781,7 +784,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; - tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval; + tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval; tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; @@ -902,7 +905,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i64; tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval; - + tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32; + GRANT_CFG_GET; return 0; } diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index f7a8ec4564..c8a6597bad 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -38,6 +38,15 @@ typedef struct { rocksdb_comparator_t** pCompares; } RocksdbCfInst; +uint32_t nextPow2(uint32_t x) { + x = x - 1; + x = x | (x >> 1); + x = x | (x >> 2); + x = x | (x >> 4); + x = x | (x >> 8); + x = x | (x >> 16); + return x + 1; +} int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t nCf); void destroyRocksdbCfInst(RocksdbCfInst* inst); @@ -92,7 +101,8 @@ void* streamBackendInit(const char* path) { rocksdb_options_set_recycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 2); rocksdb_options_set_info_log_level(opts, 0); - rocksdb_options_set_db_write_buffer_size(opts, 256 << 20); + uint32_t dbLimit = nextPow2(tsMaxStreamBackendCache); + rocksdb_options_set_db_write_buffer_size(opts, dbLimit << 20); pHandle->env = env; pHandle->dbOpt = opts; From ed851b522cc6a9da43a009cb09145bc05fd4183a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 31 May 2023 03:30:10 +0000 Subject: [PATCH 173/238] change db param --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cc368167f8..c5646d92d1 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -60,7 +60,7 @@ int32_t tsNumOfQnodeQueryThreads = 4; int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeStreamThreads = 4; int32_t tsNumOfSnodeWriteThreads = 1; -int32_t tsMaxStreamBackendCache = 256; // M +int32_t tsMaxStreamBackendCache = 128; // M // sync raft int32_t tsElectInterval = 25 * 1000; From 18b971c666873525f0f8fade035af51f5995d7eb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 17:06:38 +0800 Subject: [PATCH 174/238] fix(query): check the version range when dump partial rows of file block to sdata block directly. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 54 ++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 362934ec84..9ab8654bd9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1121,6 +1121,27 @@ static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, pReader->order); } + if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer)|| + (pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.minVer > pBlock->minVer)) { + int32_t i = endPos; + + if (asc) { + for(; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->verRange.maxVer) { + break; + } + } + } else { + for(; i < pBlock->nRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->verRange.minVer) { + break; + } + } + } + + endPos = i; + } + return endPos; } @@ -1260,10 +1281,11 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { return 0; } + // row index of dump info remain the initial position, let's find the appropriate start position. if ((pDumpInfo->rowIndex == 0 && asc) || (pDumpInfo->rowIndex == pBlock->nRow - 1 && (!asc))) { - if (asc && pReader->window.skey <= pBlock->minKey.ts) { + if (asc && pReader->window.skey <= pBlock->minKey.ts && pReader->verRange.minVer <= pBlock->minVer) { // pDumpInfo->rowIndex = 0; - } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) { + } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts && pReader->verRange.maxVer >= pBlock->maxVer) { // pDumpInfo->rowIndex = pBlock->nRow - 1; } else { // find the appropriate the start position in current block, and set it to be the current rowIndex int32_t pos = asc ? pBlock->nRow - 1 : 0; @@ -1279,6 +1301,29 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { pBlock->maxVer, pReader->idStr); return TSDB_CODE_INVALID_PARA; } + + ASSERT(pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.maxVer >= pBlock->minVer); + + // find the appropriate start position that satisfies the version requirement. + if ((pReader->verRange.maxVer >= pBlock->minVer && pReader->verRange.maxVer < pBlock->maxVer)|| + (pReader->verRange.minVer <= pBlock->maxVer && pReader->verRange.minVer > pBlock->minVer)) { + int32_t i = pDumpInfo->rowIndex; + if (asc) { + for(; i < pBlock->nRow; ++i) { + if (pBlockData->aVersion[i] >= pReader->verRange.minVer) { + break; + } + } + } else { + for(; i >= 0; --i) { + if (pBlockData->aVersion[i] <= pReader->verRange.maxVer) { + break; + } + } + } + + pDumpInfo->rowIndex = i; + } } } @@ -1293,6 +1338,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); if (dumpedRows > pReader->resBlockInfo.capacity) { // output buffer check dumpedRows = pReader->resBlockInfo.capacity; + } else if (dumpedRows <= 0) { // no qualified rows in current data block, abort directly. + setBlockAllDumped(pDumpInfo, pReader->window.ekey, pReader->order); + return TSDB_CODE_SUCCESS; } int32_t i = 0; @@ -1848,7 +1896,7 @@ static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBloc SDataBlockToLoadInfo info = {0}; getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader); bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || - info.overlapWithDelInfo || info.overlapWithLastBlock || info.partiallyRequired); + info.overlapWithDelInfo || info.overlapWithLastBlock); return isCleanFileBlock; } From fa5c024b3ee97914ae58e58f5507719eddf19977 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 17:22:59 +0800 Subject: [PATCH 175/238] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9ab8654bd9..7af0936bb5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2857,7 +2857,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // it is a clean block, load it directly if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && pBlock->nRow <= pReader->resBlockInfo.capacity) { - if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { + if (asc || (!hasDataInLastBlock(pLastBlockReader))) { code = copyBlockDataToSDataBlock(pReader); if (code) { goto _end; @@ -2876,7 +2876,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } - SBlockData* pBlockData = &pReader->status.fileBlockData; + SBlockData* pBlockData = &pReader->status.fileBlockData; while (1) { bool hasBlockData = false; @@ -2890,7 +2890,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pDumpInfo->rowIndex += step; - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + pBlock = getCurrentBlock(&pReader->status.blockIter); if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); // NOTE: get the new block info @@ -2918,7 +2918,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + pBlock = getCurrentBlock(&pReader->status.blockIter); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; } From 65b81a803f7edda2ab3b0aa50c08c5c258c6c521 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 18:05:53 +0800 Subject: [PATCH 176/238] refactor: do some internal refactor and add some logs. --- source/libs/stream/src/stream.c | 4 +++- source/libs/stream/src/streamDispatch.c | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 8bca0e6ec2..72fd520498 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -237,7 +237,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); - qDebug("task %d is shuffle, left waiting rsp %d", pTask->id.taskId, leftRsp); + qDebug("s-task:%s is shuffle, left waiting rsp %d", pTask->id.idStr, leftRsp); if (leftRsp > 0) { return 0; } @@ -246,6 +246,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); ASSERT(old == TASK_OUTPUT_STATUS__WAIT); + qDebug("s-task:%s receive dispatch rsp, output status:%d", pTask->id.idStr, pTask->outputStatus); + // the input queue of the (down stream) task that receive the output data is full, so the TASK_INPUT_STATUS_BLOCKED is rsp // todo we need to send EMPTY PACKAGE to detect if the input queue is available for output of upstream task, every 50 ms. if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 401a8b9e74..1e939cb071 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -510,14 +510,17 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { int8_t old = atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); if (old != TASK_OUTPUT_STATUS__NORMAL) { - qDebug("s-task:%s task wait for dispatch rsp, not dispatch now", pTask->id.idStr); + qDebug("s-task:%s task wait for dispatch rsp, not dispatch now, output status:%d", pTask->id.idStr, old); return 0; } + qDebug("s-task:%s start to dispatch msg, output status:%d", pTask->id.idStr, pTask->outputStatus); + SStreamDataBlock* pDispatchedBlock = streamQueueNextItem(pTask->outputQueue); if (pDispatchedBlock == NULL) { - qDebug("s-task:%s stop dispatching since no output in output queue", pTask->id.idStr); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); + qDebug("s-task:%s stop dispatching since no output in output queue, output status:%d", pTask->id.idStr, + pTask->outputStatus); return 0; } @@ -527,6 +530,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { streamQueueProcessFail(pTask->outputQueue); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); + qDebug("s-task:%s failed to dispatch msg to downstream, output status:%d", pTask->id.idStr, pTask->outputStatus); } // this block can be freed only when it has been pushed to down stream. From 60100a0fb6dd54f920bf8a5535b0cce01d80c9f3 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 31 May 2023 15:34:32 +0800 Subject: [PATCH 177/238] feature: add user priv when create db, remove priv check with create_dbs --- source/dnode/mnode/impl/inc/mndUser.h | 2 + source/dnode/mnode/impl/src/mndDb.c | 22 ++++- source/dnode/mnode/impl/src/mndUser.c | 4 +- source/libs/catalog/src/ctgUtil.c | 3 + .../script/tsim/user/privilege_create_db.sim | 97 +++++++++++++++++++ tests/script/win-test-file | 1 + 6 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 tests/script/tsim/user/privilege_create_db.sim diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index aa7f97f087..93ae38e554 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -40,6 +40,8 @@ int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int3 int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); +int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew); +void mndUserFreeObj(SUserObj *pUser); #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index a451ae9df5..70214e80a5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -446,7 +446,8 @@ static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } -static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups, + SUserObj *pUserDuped) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); if (pDbRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; @@ -459,6 +460,13 @@ static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj * if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } + if (pUserDuped) { + SSdbRaw *pUserRaw = mndUserActionEncode(pUserDuped); + if (pUserRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pUserRaw) != 0) return -1; + if (sdbSetRawStatus(pUserRaw, SDB_STATUS_READY) != 0) return -1; + } + return 0; } @@ -565,6 +573,15 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, return -1; } + // add database privileges for user + SUserObj newUserObj = {0}, *pNewUserDuped = NULL; + if (!pUser->superUser) { + if (mndUserDupObj(pUser, &newUserObj) != 0) goto _OVER; + taosHashPut(newUserObj.readDbs, dbObj.name, strlen(dbObj.name) + 1, dbObj.name, TSDB_FILENAME_LEN); + taosHashPut(newUserObj.writeDbs, dbObj.name, strlen(dbObj.name) + 1, dbObj.name, TSDB_FILENAME_LEN); + pNewUserDuped = &newUserObj; + } + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "create-db"); if (pTrans == NULL) goto _OVER; @@ -577,7 +594,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, mndTransSetOper(pTrans, MND_OPER_CREATE_DB); if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; - if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; + if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups, pNewUserDuped) != 0) goto _OVER; if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -586,6 +603,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, _OVER: taosMemoryFree(pVgroups); + mndUserFreeObj(&newUserObj); mndTransDrop(pTrans); return code; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 3da594109a..90d16a0a81 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -488,7 +488,7 @@ SHashObj *mndDupUseDbHash(SHashObj *pOld) { return pNew; } -static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { +int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { memcpy(pNew, pUser, sizeof(SUserObj)); pNew->authVersion++; pNew->updateTime = taosGetTimestampMs(); @@ -508,7 +508,7 @@ static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { return 0; } -static void mndUserFreeObj(SUserObj *pUser) { +void mndUserFreeObj(SUserObj *pUser) { taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->topics); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index cf864e8643..6f4347a49c 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1555,10 +1555,13 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(&pReq->tbName, dbFName); + // since that we add read/write previliges when create db, there is no need to check createdDbs +#if 0 if (pInfo->createdDbs && taosHashGet(pInfo->createdDbs, dbFName, strlen(dbFName))) { pRes->pass = true; return TSDB_CODE_SUCCESS; } +#endif switch (pReq->type) { case AUTH_TYPE_READ: { diff --git a/tests/script/tsim/user/privilege_create_db.sim b/tests/script/tsim/user/privilege_create_db.sim new file mode 100644 index 0000000000..c81bd1b258 --- /dev/null +++ b/tests/script/tsim/user/privilege_create_db.sim @@ -0,0 +1,97 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ========================root user create user +sql create user u1 pass "taosdata" +sql create user u2 pass "taosdata" +sql create database test +sql select * from information_schema.ins_user_privileges where user_name == "root" +if $rows != 1 then + return -1 +endi + +print =============connect with u1 +sql connect u1 +sql create database u1_d1 +sql use u1_d1 +sql create table u1_d1.t1(ts timestamp, c2 int) +sql use information_schema +sql select * from ins_user_privileges where user_name == "u1" order by privilege +if $rows != 2 then + return -1 +endi +if $data01 != read then + return -1 +endi +if $data11 != write then + return -1 +endi +if $data02 != u1_d1 then + return -1 +endi +if $data12 != u1_d1 then + return -1 +endi + +sql_error grant all on *.* to u1 +sql_error grant all on test.* to u1 + +print =============connect with u2 +sql connect u2 +sql create database u2_d1 +sql use u2_d1 +sql create table u2_d1.t1(ts timestamp, c2 int) +sql use information_schema +sql select * from ins_user_privileges where user_name == "u2" order by privilege +if $rows != 2 then + return -1 +endi +if $data01 != read then + return -1 +endi +if $data11 != write then + return -1 +endi +if $data02 != u2_d1 then + return -1 +endi +if $data12 != u2_d1 then + return -1 +endi + +sql_error select * from u1_d1.t1 +sql_error revoke read on u2_d1.* from u2 + +print =============connect with root, revoke read from u1, all from u2 +sql connect +sql revoke read on u1_d1.* from u1 +sql revoke all on u2_d1.* from u2 +sleep 1000 + +print =============connect with u1 +sql connect u1 +sql insert into u1_d1.t1 values(now, 1) +sql_error select * from u1_d1.t1; + +print =============connect with u2 +sql connect u2 +sql_error select * from u2_d1.t1; +sql_error insert into u2_d1.t1 values(now, 1) + +print =============connect with root, grant read to u1, all to u2 +sql connect +sql grant read on u1_d1.* to u1 +sql grant all on u2_d1.* to u2 + +sleep 1000 +print =============connect with u1 +sql connect u1 +sql select * from u1_d1.t1; +sql insert into u1_d1.t1 values(now, 2) + +print =============connect with u2 +sql connect u2 +sql select * from u2_d1.t1; +sql insert into u2_d1.t1 values(now, 2) diff --git a/tests/script/win-test-file b/tests/script/win-test-file index b7fbbed5c1..adef71cb45 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -4,6 +4,7 @@ ./test.sh -f tsim/user/privilege_sysinfo.sim ./test.sh -f tsim/user/privilege_topic.sim ./test.sh -f tsim/user/privilege_table.sim +./test.sh -f tsim/user/privilege_create_db.sim ./test.sh -f tsim/db/alter_option.sim rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/db/alter_replica_31.sim From a3abe25539a07ca87fb051cdbf5fe4056aae8440 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:03:37 +0800 Subject: [PATCH 178/238] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 6 ++++-- source/dnode/vnode/src/tq/tqSink.c | 12 +++++------- source/libs/stream/src/stream.c | 10 ++++++---- source/libs/stream/src/streamExec.c | 2 +- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index dcfc578ac7..917f06ae14 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1272,13 +1272,15 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = ntohl(pRsp->upstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - tqDebug("recv dispatch rsp, code:%x", pMsg->code); + + int32_t vgId = pTq->pStreamMeta->vgId; + tqDebug("vgId:%d recv dispatch rsp, code:%d", vgId, pMsg->code); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } else { - return -1; + tqDebug("vgId:%d failed to find task:0x%x", vgId, taskId); } } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 0bd7d9a57b..6e02d5e21b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -137,7 +137,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d int32_t blockSz = taosArrayGetSize(pBlocks); - tqDebug("vgId:%d, s-task:%s write results blocks:%d into table", TD_VID(pVnode), pTask->id.idStr, blockSz); + tqDebug("vgId:%d, s-task:%s write results %d blocks into table", TD_VID(pVnode), pTask->id.idStr, blockSz); void* pBuf = NULL; SArray* tagArray = NULL; @@ -482,17 +482,15 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d tEncoderClear(&encoder); tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); - SRpcMsg msg = { - .msgType = TDMT_VND_SUBMIT, - .pCont = pBuf, - .contLen = len, - }; - + SRpcMsg msg = { .msgType = TDMT_VND_SUBMIT, .pCont = pBuf, .contLen = len }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { tqDebug("failed to put into write-queue since %s", terrstr()); } } } + + tqDebug("vgId:%d, s-task:%s write results completed", TD_VID(pVnode), pTask->id.idStr); + _end: taosArrayDestroy(tagArray); taosArrayDestroy(pVals); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 72fd520498..acc69c5a2b 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -126,7 +126,7 @@ int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pR if (pBlock == NULL) { streamTaskInputFail(pTask); status = TASK_INPUT_STATUS__FAILED; - qDebug("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId, + qError("vgId:%d, s-task:%s failed to receive dispatch msg, reason: out of memory", pTask->pMeta->vgId, pTask->id.idStr); } else { int32_t code = tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pBlock); @@ -233,7 +233,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { - qDebug("s-task:%s receive dispatch rsp, status:%d code:%d", pTask->id.idStr, pRsp->inputStatus, code); + qDebug("s-task:%s receive dispatch rsp, output status:%d code:%d", pTask->id.idStr, pRsp->inputStatus, code); if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); @@ -246,13 +246,15 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); ASSERT(old == TASK_OUTPUT_STATUS__WAIT); - qDebug("s-task:%s receive dispatch rsp, output status:%d", pTask->id.idStr, pTask->outputStatus); - // the input queue of the (down stream) task that receive the output data is full, so the TASK_INPUT_STATUS_BLOCKED is rsp // todo we need to send EMPTY PACKAGE to detect if the input queue is available for output of upstream task, every 50 ms. if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { // TODO: init recover timer qError("s-task:%s inputQ of downstream task:0x%x is full, need to block output", pTask->id.idStr, pRsp->downstreamTaskId); + + atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); + qError("s-task:%s ignore error, and reset task output status:%d", pTask->id.idStr, pTask->outputStatus); + return 0; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 95b97e080a..d6f5533db4 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -385,7 +385,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { if (pTask->taskLevel == TASK_LEVEL__SINK) { ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); - qDebug("s-task:%s sink node start to sink result. numOfBlocks:%d", pTask->id.idStr, batchSize); + qDebug("s-task:%s sink task start to sink %d blocks", pTask->id.idStr, batchSize); streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pInput); continue; } From 9e021087a935c79bf56bef5e498e84427a7cdb45 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:04:53 +0800 Subject: [PATCH 179/238] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 917f06ae14..6ec9020759 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1281,6 +1281,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { return 0; } else { tqDebug("vgId:%d failed to find task:0x%x", vgId, taskId); + return TSDB_CODE_INVALID_MSG; } } From 04f3784696cc11f534451196b2aee4a9f2ea9d98 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 31 May 2023 11:05:03 +0000 Subject: [PATCH 180/238] change log level --- source/libs/stream/src/streamBackendRocksdb.c | 14 +++++++------- source/libs/stream/src/streamDispatch.c | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c8a6597bad..9d94a28c8a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1026,10 +1026,10 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ if (err != NULL) { \ taosMemoryFree(err); \ - qDebug("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ + qError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ code = -1; \ } else { \ - qDebug("streamState str:%s succ to write to %s, valLen:%d", toString, funcname, vLen); \ + qTrace("streamState str:%s succ to write to %s, valLen:%d", toString, funcname, vLen); \ } \ taosMemoryFree(ttlV); \ } while (0); @@ -1056,10 +1056,10 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ if (val == NULL) { \ if (err == NULL) { \ - qDebug("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ + qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ funcname); \ } else { \ - qDebug("streamState str: %s failed to read from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ + qError("streamState str: %s failed to read from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ err); \ taosMemoryFreeClear(err); \ } \ @@ -1068,11 +1068,11 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa char* p = NULL; \ int32_t len = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ if (len < 0) { \ - qDebug("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ + qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ funcname); \ code = -1; \ } else { \ - qDebug("streamState str: %s succ to read from %s_%s, valLen:%d", toString, pState->pTdbState->idstr, funcname, \ + qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, pState->pTdbState->idstr, funcname, \ len); \ } \ taosMemoryFree(val); \ @@ -1107,7 +1107,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa taosMemoryFree(err); \ code = -1; \ } else { \ - qDebug("streamState str: %s succ to del from %s_%s", toString, pState->pTdbState->idstr, funcname); \ + qTrace("streamState str: %s succ to del from %s_%s", toString, pState->pTdbState->idstr, funcname); \ } \ } while (0); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 401a8b9e74..042ea373a1 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -157,7 +157,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) tEncodeStreamRetrieveReq(&encoder, &req); tEncoderClear(&encoder); - SRpcMsg rpcMsg = { .code = 0, .msgType = TDMT_STREAM_RETRIEVE, .pCont = buf, .contLen = sizeof(SMsgHead) + len }; + SRpcMsg rpcMsg = {.code = 0, .msgType = TDMT_STREAM_RETRIEVE, .pCont = buf, .contLen = sizeof(SMsgHead) + len}; if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) { ASSERT(0); goto CLEAR; From 6d04c4e2fd3ab3463c422f73cb298a51a9b586c2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:05:40 +0800 Subject: [PATCH 181/238] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6ec9020759..8cc791d1a6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1274,13 +1274,12 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); int32_t vgId = pTq->pStreamMeta->vgId; - tqDebug("vgId:%d recv dispatch rsp, code:%d", vgId, pMsg->code); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } else { - tqDebug("vgId:%d failed to find task:0x%x", vgId, taskId); + tqDebug("vgId:%d failed to handle the dispatch rsp, since find task:0x%x failed", vgId, taskId); return TSDB_CODE_INVALID_MSG; } } From 87deb36f8a3a6fe9c903e5be0f775478a0d15149 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:36:02 +0800 Subject: [PATCH 182/238] refactor: do some internal refactor. --- source/libs/executor/src/executor.c | 5 ++++- source/libs/executor/src/scanoperator.c | 10 ++++++---- source/libs/stream/src/streamExec.c | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fb35b211c9..ed8a4f3a3b 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -132,7 +132,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pOperator->status = OP_NOT_OPENED; SStreamScanInfo* pInfo = pOperator->info; - qDebug("s-task:%s set source blocks:%d", id, (int32_t)numOfBlocks); + + qDebug("s-task:%s in this batch, all %d blocks need to be processed and dump results", id, (int32_t)numOfBlocks); ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0); if (type == STREAM_INPUT__MERGED_SUBMIT) { @@ -140,6 +141,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); taosArrayPush(pInfo->pBlockLists, pReq); } + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { taosArrayPush(pInfo->pBlockLists, input); @@ -150,6 +152,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SPackedData tmp = { .pDataBlock = pDataBlock }; taosArrayPush(pInfo->pBlockLists, &tmp); } + pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } else { ASSERT(0); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2d2f377041..88f5642ef9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1797,9 +1797,10 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + const char* id = GET_TASKID(pTaskInfo); + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SStreamScanInfo* pInfo = pOperator->info; qDebug("stream scan started, %s", GET_TASKID(pTaskInfo)); @@ -1922,7 +1923,9 @@ FETCH_NEXT_BLOCK: return NULL; } - int32_t current = pInfo->validBlockIndex++; + int32_t current = pInfo->validBlockIndex++; + qDebug("process %d/%d input data blocks, %s", current, (int32_t) total, id); + SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.parTbName[0]) { @@ -2057,7 +2060,6 @@ FETCH_NEXT_BLOCK: return pInfo->pUpdateRes; } - const char* id = GET_TASKID(pTaskInfo); SSDataBlock* pBlock = pInfo->pRes; SDataBlockInfo* pBlockInfo = &pBlock->info; int32_t totalBlocks = taosArrayGetSize(pInfo->pBlockLists); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index d6f5533db4..0970bbcf0c 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -16,8 +16,8 @@ #include "streamInc.h" // maximum allowed processed block batches. One block may include several submit blocks -#define MAX_STREAM_EXEC_BATCH_NUM 128 -#define MIN_STREAM_EXEC_BATCH_NUM 16 +#define MAX_STREAM_EXEC_BATCH_NUM 32 +#define MIN_STREAM_EXEC_BATCH_NUM 8 #define MAX_STREAM_RESULT_DUMP_THRESHOLD 1000 static int32_t updateCheckPointInfo (SStreamTask* pTask); From ccc86f991637e101a1544a5a409fca5049b39740 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 19:36:34 +0800 Subject: [PATCH 183/238] refactor: do some internal refactor. --- source/libs/stream/src/streamExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 0970bbcf0c..716b939e5f 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -18,7 +18,7 @@ // maximum allowed processed block batches. One block may include several submit blocks #define MAX_STREAM_EXEC_BATCH_NUM 32 #define MIN_STREAM_EXEC_BATCH_NUM 8 -#define MAX_STREAM_RESULT_DUMP_THRESHOLD 1000 +#define MAX_STREAM_RESULT_DUMP_THRESHOLD 100 static int32_t updateCheckPointInfo (SStreamTask* pTask); From 91de00597d64d89b41acb254eb09f30fdf67c874 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 31 May 2023 22:12:14 +0800 Subject: [PATCH 184/238] refactor: do some internal refactor. --- source/libs/stream/src/streamExec.c | 36 ++++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 716b939e5f..c3dd848bc7 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -17,7 +17,7 @@ // maximum allowed processed block batches. One block may include several submit blocks #define MAX_STREAM_EXEC_BATCH_NUM 32 -#define MIN_STREAM_EXEC_BATCH_NUM 8 +#define MIN_STREAM_EXEC_BATCH_NUM 4 #define MAX_STREAM_RESULT_DUMP_THRESHOLD 100 static int32_t updateCheckPointInfo (SStreamTask* pTask); @@ -44,6 +44,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* if (numOfBlocks > 0) { SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(pItem, pTask, size, pRes); if (pStreamBlocks == NULL) { + qError("s-task:%s failed to create result stream data block, code:%s", pTask->id.idStr, tstrerror(terrno)); taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return -1; } @@ -314,8 +315,13 @@ int32_t updateCheckPointInfo (SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } +/** + * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the + * appropriate batch of blocks should be handled in 5 to 10 sec. + */ int32_t streamExecForAll(SStreamTask* pTask) { - int32_t code = 0; + const char* id = pTask->id.idStr; + while (1) { int32_t batchSize = 1; int16_t times = 0; @@ -323,7 +329,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { SStreamQueueItem* pInput = NULL; // merge multiple input data if possible in the input queue. - qDebug("s-task:%s start to extract data block from inputQ", pTask->id.idStr); + qDebug("s-task:%s start to extract data block from inputQ", id); while (1) { if (streamTaskShouldPause(&pTask->status)) { @@ -338,7 +344,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { if (qItem == NULL) { if (pTask->taskLevel == TASK_LEVEL__SOURCE && batchSize < MIN_STREAM_EXEC_BATCH_NUM && times < 5) { times++; - taosMsleep(1); + taosMsleep(10); qDebug("===stream===try again batchSize:%d", batchSize); continue; } @@ -363,8 +369,10 @@ int32_t streamExecForAll(SStreamTask* pTask) { batchSize++; pInput = newRet; streamQueueProcessSuccess(pTask->inputQueue); + if (batchSize > MAX_STREAM_EXEC_BATCH_NUM) { - qDebug("maximum batch limit:%d reached, processing, %s", MAX_STREAM_EXEC_BATCH_NUM, pTask->id.idStr); + qDebug("s-task:%s maximum batch limit:%d reached, processing this batch of blocks", id, + MAX_STREAM_EXEC_BATCH_NUM); break; } } @@ -375,7 +383,6 @@ int32_t streamExecForAll(SStreamTask* pTask) { if (pInput) { streamFreeQitem(pInput); } - return 0; } @@ -385,7 +392,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { if (pTask->taskLevel == TASK_LEVEL__SINK) { ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); - qDebug("s-task:%s sink task start to sink %d blocks", pTask->id.idStr, batchSize); + qDebug("s-task:%s sink task start to sink %d blocks", id, batchSize); streamTaskOutputResultBlock(pTask, (SStreamDataBlock*)pInput); continue; } @@ -394,16 +401,16 @@ int32_t streamExecForAll(SStreamTask* pTask) { while (pTask->taskLevel == TASK_LEVEL__SOURCE) { int8_t status = atomic_load_8(&pTask->status.taskStatus); if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__PAUSE) { - qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr, + qError("stream task wait for the end of fill history, s-task:%s, status:%d", id, atomic_load_8(&pTask->status.taskStatus)); - taosMsleep(2); + taosMsleep(100); } else { break; } } int64_t st = taosGetTimestampMs(); - qDebug("s-task:%s start to execute, block batches:%d", pTask->id.idStr, batchSize); + qDebug("s-task:%s start to process batch of blocks, num:%d", id, batchSize); { // set input @@ -417,21 +424,21 @@ int32_t streamExecForAll(SStreamTask* pTask) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); - qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, + qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, id, pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)pInput; SArray* pBlockList = pBlock->blocks; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); + qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, id, numOfBlocks, pBlock->sourceVer); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput; SArray* pBlockList = pMerged->submits; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks); + qDebug("s-task:%s %p set submit input (merged), batch num:%d", id, pTask, numOfBlocks); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput; @@ -446,7 +453,8 @@ int32_t streamExecForAll(SStreamTask* pTask) { streamTaskExecImpl(pTask, pInput, &resSize, &totalBlocks); double el = (taosGetTimestampMs() - st) / 1000.0; - qDebug("s-task:%s exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", pTask->id.idStr, el, resSize / 1048576.0, totalBlocks); + qDebug("s-task:%s batch of input blocks exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", + id, el, resSize / 1048576.0, totalBlocks); streamFreeQitem(pInput); } From fb2f367aacd17f9128996097e4b074f37dfe2199 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 1 Jun 2023 08:13:32 +0800 Subject: [PATCH 185/238] meta/split: trim invalidated uids --- source/dnode/vnode/src/inc/vnodeInt.h | 16 +++---- source/dnode/vnode/src/meta/metaTable.c | 55 ++++++++++++++++++++++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 17 ++++---- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index db285dc124..0bfdf33019 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -107,10 +107,10 @@ struct SQueryNode { typedef SVCreateTbReq STbCfg; typedef SVCreateTSmaReq SSmaCfg; -SMTbCursor *metaOpenTbCursor(void *pVnode); -void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); +SMTbCursor* metaOpenTbCursor(void* pVnode); +void metaCloseTbCursor(SMTbCursor* pTbCur); +int32_t metaTbCursorNext(SMTbCursor* pTbCur, ETableType jumpTableType); +int32_t metaTbCursorPrev(SMTbCursor* pTbCur, ETableType jumpTableType); #endif @@ -146,6 +146,7 @@ int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* p int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); +int32_t metaTrimTables(SMeta* pMeta); int metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); @@ -154,8 +155,8 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); -int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); -int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); +int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); +int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); @@ -175,7 +176,7 @@ void* metaGetIdx(SMeta* pMeta); void* metaGetIvtIdx(SMeta* pMeta); int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList); -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void metaReaderInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -491,7 +492,6 @@ struct SCompactInfo { void initStorageAPI(SStorageAPI* pAPI); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 4f9f0c688e..20cf423182 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -848,7 +848,52 @@ static void metaDropTables(SMeta *pMeta, SArray *tbUids) { metaULock(pMeta); } -int metaTrimTables(SMeta *pMeta, int64_t version) { +static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { + int32_t code = 0; + // 1, tranverse table's + // 2, validate table name using vnodeValidateTableHash + // 3, push invalidated table's uid into uidList + + TBC *pCur; + code = tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); + if (code < 0) { + return code; + } + + code = tdbTbcMoveToFirst(pCur); + if (code) { + tdbTbcClose(pCur); + return code; + } + + void *pData = NULL; + int nData = 0; + + while (1) { + int32_t ret = tdbTbcNext(pCur, NULL, NULL, &pData, &nData); + if (ret < 0) { + break; + } + + SMetaEntry me = {0}; + SDecoder dc = {0}; + tDecoderInit(&dc, pData, nData); + metaDecodeEntry(&dc, &me); + if (me.type == TSDB_CHILD_TABLE) { + int32_t ret = vnodeValidateTableHash(pMeta->pVnode, me.name); + if (TSDB_CODE_VND_HASH_MISMATCH == ret) { + taosArrayPush(uidList, &me.uid); + } + } + tDecoderClear(&dc); + } + tdbFree(pData); + tdbTbcClose(pCur); + + return 0; +} + +int32_t metaTrimTables(SMeta *pMeta) { int32_t code = 0; SArray *tbUids = taosArrayInit(8, sizeof(int64_t)); @@ -856,7 +901,7 @@ int metaTrimTables(SMeta *pMeta, int64_t version) { return TSDB_CODE_OUT_OF_MEMORY; } - // code = metaFilterTableByHash(pMeta, /*ttl, */ tbUids); + code = metaFilterTableByHash(pMeta, tbUids); if (code != 0) { goto end; } @@ -1027,7 +1072,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1); metaUidCacheClear(pMeta, e.ctbEntry.suid); - metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) @@ -1039,7 +1084,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaStatsCacheDrop(pMeta, uid); metaUidCacheClear(pMeta, uid); - metaTbGroupCacheClear(pMeta, uid); + metaTbGroupCacheClear(pMeta, uid); --pMeta->pVnode->config.vndStats.numOfSTables; } @@ -1460,7 +1505,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid); - metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid); + metaTbGroupCacheClear(pMeta, ctbEntry.ctbEntry.suid); metaULock(pMeta); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b950437f23..c3fb5e5ad4 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -245,11 +245,11 @@ _exit: static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; - int32_t size; - int32_t ret; - uint8_t *pCont; - SEncoder *pCoder = &(SEncoder){0}; - SDeleteRes res = {0}; + int32_t size; + int32_t ret; + uint8_t *pCont; + SEncoder *pCoder = &(SEncoder){0}; + SDeleteRes res = {0}; SReadHandle handle = {.config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; initStorageAPI(&handle.api); @@ -316,8 +316,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg return -1; } - vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), - ver); + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), ver); ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); ASSERT(pVnode->state.applied + 1 == ver); @@ -1479,6 +1478,7 @@ static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) { pVnode->config.hashBegin, pVnode->config.hashEnd, ver); // TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd] + code = metaTrimTables(pVnode->pMeta); return code; } @@ -1492,8 +1492,7 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pR code = vnodeConsolidateAlterHashRange(pVnode, ver); if (code < 0) { - vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(), - ver); + vError("vgId:%d, failed to consolidate alter hashrange since %s. version:%" PRId64, TD_VID(pVnode), terrstr(), ver); goto _exit; } pVnode->config.hashChange = false; From 9b9bf2de6897a1453dfc31b9a13e77125a606d09 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 1 Jun 2023 10:41:01 +0800 Subject: [PATCH 186/238] meta/tbc: key cannot be null --- source/dnode/vnode/src/meta/metaTable.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 20cf423182..eb169fbdc2 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -866,11 +866,11 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { return code; } - void *pData = NULL; - int nData = 0; + void *pData = NULL, *pKey = NULL; + int nData = 0, nKey = 0; while (1) { - int32_t ret = tdbTbcNext(pCur, NULL, NULL, &pData, &nData); + int32_t ret = tdbTbcNext(pCur, &pKey, &nKey, &pData, &nData); if (ret < 0) { break; } @@ -879,7 +879,7 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { SDecoder dc = {0}; tDecoderInit(&dc, pData, nData); metaDecodeEntry(&dc, &me); - if (me.type == TSDB_CHILD_TABLE) { + if (me.type != TSDB_SUPER_TABLE) { int32_t ret = vnodeValidateTableHash(pMeta->pVnode, me.name); if (TSDB_CODE_VND_HASH_MISMATCH == ret) { taosArrayPush(uidList, &me.uid); @@ -888,6 +888,7 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { tDecoderClear(&dc); } tdbFree(pData); + tdbFree(pKey); tdbTbcClose(pCur); return 0; From 6517ede04461b5f471c1859fbbd7d3963e0010df Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 1 Jun 2023 11:23:31 +0800 Subject: [PATCH 187/238] fix: fix interval operator ensure capacity logic --- source/libs/executor/src/executorInt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index e18906ad18..5872d019fa 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -827,7 +827,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - uint32_t newSize = pBlock->info.rows + pRow->numOfRows + (numOfRows - i) > 1 ? 1 : 0; + uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - i) > 1 ? 1 : 0); blockDataEnsureCapacity(pBlock, newSize); qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", newSize, pBlock->info.capacity, GET_TASKID(pTaskInfo)); From 706e35021733713389bdb065d064189e4fb89502 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jun 2023 14:36:42 +0800 Subject: [PATCH 188/238] fix(query): fix error in fill and partition. --- source/libs/executor/inc/tfill.h | 2 ++ source/libs/executor/src/filloperator.c | 28 +++++++++++++++++++------ source/libs/executor/src/tfill.c | 19 ++++++++++++++--- tests/script/tsim/query/partitionby.sim | 4 ++-- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 78b3cd2f40..79837480d7 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -120,6 +120,8 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts); +bool taosFillNotStarted(const SFillInfo* pFillInfo); SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, const struct SNodeListNode* val); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 839c845bac..3d1df6482e 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -139,6 +139,12 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { } SOperatorInfo* pDownstream = pOperator->pDownstream[0]; + + // the scan order may be different from the output result order for agg interval operator. + if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL) { + order = ((SIntervalAggOperatorInfo*) pDownstream->info)->resultTsOrder; + } + while (1) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { @@ -162,11 +168,24 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block pInfo->totalInputRows += pInfo->pRes->info.rows; - if (order == pInfo->pFillInfo->order) { + if (order == TSDB_ORDER_ASC) { + int64_t skey = pBlock->info.window.skey; + if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); + } + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); } else { + int64_t ekey = pBlock->info.window.ekey; + if (ekey > pInfo->pFillInfo->start) { + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); + } + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); } + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block pInfo->existNewGroupBlock = pBlock; @@ -256,11 +275,8 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); - STimeWindow w = {0}; - int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - - getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); - pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, + int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; + pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); if (order == TSDB_ORDER_ASC) { diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index e4d5d2c268..7cc50a70ab 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -517,11 +517,16 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) return; } + // the endKey is now the aligned time window value. truncate time window isn't correct. pFillInfo->end = endKey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval); - pFillInfo->end = taosTimeAdd(pFillInfo->end, pFillInfo->interval.interval, pFillInfo->interval.intervalUnit,pFillInfo->interval.precision); + +#if 0 + if (pFillInfo->order == TSDB_ORDER_ASC) { + ASSERT(pFillInfo->start <= pFillInfo->end); + } else { + ASSERT(pFillInfo->start >= pFillInfo->end); } +#endif pFillInfo->index = 0; pFillInfo->numOfRows = numOfRows; @@ -531,6 +536,13 @@ void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) pFillInfo->pSrcBlock = (SSDataBlock*)pInput; } +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts) { + pFillInfo->start = ts; + pFillInfo->currentKey = ts; +} + +bool taosFillNotStarted(const SFillInfo* pFillInfo) {return pFillInfo->start == pFillInfo->currentKey;} + bool taosFillHasMoreResults(SFillInfo* pFillInfo) { int32_t remain = taosNumOfRemainRows(pFillInfo); if (remain > 0) { @@ -565,6 +577,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } + numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); numOfRes += 1; diff --git a/tests/script/tsim/query/partitionby.sim b/tests/script/tsim/query/partitionby.sim index da7dfc0090..76d4f87908 100644 --- a/tests/script/tsim/query/partitionby.sim +++ b/tests/script/tsim/query/partitionby.sim @@ -121,7 +121,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 2 then +if $data00 != 4 then return -1 endi @@ -129,7 +129,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 8 then +if $data00 != 16 then return -1 endi From 21920721f3b7c34924d6c9971a8e8677618eb413 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 15:10:08 +0800 Subject: [PATCH 189/238] fix:make version compatible --- source/dnode/mnode/impl/src/mndTopic.c | 8 +++++--- source/dnode/vnode/src/tq/tqMeta.c | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 10bd7a7a8d..a2c0014245 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -28,7 +28,7 @@ #include "parser.h" #include "tname.h" -#define MND_TOPIC_VER_NUMBER 2 +#define MND_TOPIC_VER_NUMBER 3 #define MND_TOPIC_RESERVE_SIZE 64 SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); @@ -170,7 +170,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER; - if (sver != 1 && sver != 2) { + if (sver < 1 || sver > MND_TOPIC_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto TOPIC_DECODE_OVER; } @@ -197,7 +197,9 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pTopic->withMeta, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->stbUid, TOPIC_DECODE_OVER); - SDB_GET_BINARY(pRaw, dataPos, pTopic->stbName, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER); + if (sver >= 3) { + SDB_GET_BINARY(pRaw, dataPos, pTopic->stbName, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER); + } SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); if (pTopic->sql == NULL) { diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 0edffd7f05..0021aa6ba7 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -65,7 +65,9 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tDecodeI64(pDecoder, &pHandle->execHandle.execTb.suid) < 0) return -1; - if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1; + if (!tDecodeIsEnd(pDecoder)){ + if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execTb.qmsg) < 0) return -1; + } } tEndDecode(pDecoder); return 0; From 64edab63025818aa807737cab687f31d0c9206c9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jun 2023 07:51:52 +0000 Subject: [PATCH 190/238] refactor code --- source/libs/stream/src/streamBackendRocksdb.c | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 9d94a28c8a..eabec058d7 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1134,31 +1134,29 @@ int32_t streamStateDel_rocksdb(SStreamState* pState, const SWinKey* key) { int32_t streamStateClear_rocksdb(SStreamState* pState) { qDebug("streamStateClear_rocksdb"); - SStateKey sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number}; - SStateKey eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number}; char sKeyStr[128] = {0}; char eKeyStr[128] = {0}; + SStateKey sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number}; + SStateKey eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number}; int sLen = stateKeyEncode(&sKey, sKeyStr); int eLen = stateKeyEncode(&eKey, eKeyStr); - char toStringStart[128] = {0}; - char toStringEnd[128] = {0}; - if (qDebugFlag & DEBUG_TRACE) { - stateKeyToString(&sKey, toStringStart); - stateKeyToString(&eKey, toStringEnd); - } - - char* err = NULL; if (pState->pTdbState->pHandle[1] != NULL) { + char* err = NULL; rocksdb_delete_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->writeOpts, pState->pTdbState->pHandle[1], sKeyStr, sLen, eKeyStr, eLen, &err); - } - // rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[0], sKeyStr, sLen, eKeyStr, - // eLen); - if (err != NULL) { - qWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); - taosMemoryFree(err); + if (err != NULL) { + char toStringStart[128] = {0}; + char toStringEnd[128] = {0}; + stateKeyToString(&sKey, toStringStart); + stateKeyToString(&eKey, toStringEnd); + + qWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); + taosMemoryFree(err); + } else { + rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[1], sKeyStr, sLen, eKeyStr, eLen); + } } return 0; From d20e0a06f6849e40bcfd74346729084009bfc3af Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 16:04:45 +0800 Subject: [PATCH 191/238] fix:make version compatible --- source/dnode/vnode/src/tq/tqMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 0021aa6ba7..25cfad4b36 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -341,7 +341,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - if(strcmp(handle.execHandle.execTb.qmsg, "") != 0) { + if(handle.execHandle.execTb.qmsg != NULL && strcmp(handle.execHandle.execTb.qmsg, "") != 0) { if (nodesStringToNode(handle.execHandle.execTb.qmsg, &handle.execHandle.execTb.node) != 0) { tqError("nodesStringToNode error in sub stable, since %s", terrstr()); return -1; From 69499b95826c9c973cb67a023a7ba8f67a6f17e4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 16:09:17 +0800 Subject: [PATCH 192/238] fix:make version compatible --- source/dnode/vnode/src/tq/tqMeta.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 25cfad4b36..239d65fdb7 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -37,7 +37,9 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { } } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (tEncodeI64(pEncoder, pHandle->execHandle.execTb.suid) < 0) return -1; - if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1; + if (pHandle->execHandle.execTb.qmsg != NULL){ + if (tEncodeCStr(pEncoder, pHandle->execHandle.execTb.qmsg) < 0) return -1; + } } tEndEncode(pEncoder); return pEncoder->pos; From 64656a59d9a9f4fb45d3355cd927093900c75620 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Thu, 1 Jun 2023 16:28:32 +0800 Subject: [PATCH 193/238] reset sream fill block index --- source/libs/executor/src/filloperator.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 90b7f4a77e..519ddc1916 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -1002,9 +1002,10 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) { SSDataBlock* pBlock = pInfo->pSrcBlock; uint64_t groupId = pBlock->info.id.groupId; SSDataBlock* pRes = pInfo->pRes; + SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); + TSKEY* tsCol = (TSKEY*)pTsCol->pData; pRes->info.id.groupId = groupId; - SColumnInfoData* pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol); - TSKEY* tsCol = (TSKEY*)pTsCol->pData; + pInfo->srcRowIndex++; if (pInfo->srcRowIndex == 0) { keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize); @@ -1242,7 +1243,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { SSDataBlock* fillResult = NULL; SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { - if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows) { + if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) { // If there are delete datablocks, we receive them first. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -1281,7 +1282,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { case STREAM_PULL_DATA: { doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock); memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - pInfo->srcRowIndex = 0; + pInfo->srcRowIndex = -1; } break; case STREAM_CREATE_CHILD_TABLE: { return pBlock; @@ -1497,7 +1498,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi goto _error; } - pInfo->srcRowIndex = 0; + pInfo->srcRowIndex = -1; setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = From bb772e2062c5b17e73a7944dbe67600868aedb2e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 17:03:02 +0800 Subject: [PATCH 194/238] fix:make version compatible --- source/dnode/mnode/impl/src/mndTopic.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index a2c0014245..f1ee7bca3b 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -924,13 +924,12 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl }else if(pTopic->subType == TOPIC_SUB_TYPE__TABLE){ SStbObj *pStb = mndAcquireStb(pMnode, pTopic->stbName); if (pStb == NULL) { - terrno = TSDB_CODE_MND_STB_NOT_EXIST; - taosMemoryFree(schemaJson); - return -1; + STR_TO_VARSTR(schemaJson, "NULL"); + mError("mndRetrieveTopic mndAcquireStb null stbName:%s", pTopic->stbName); + }else{ + schemaToJson(pStb->pColumns, pStb->numOfColumns, schemaJson); + mndReleaseStb(pMnode, pStb); } - schemaToJson(pStb->pColumns, pStb->numOfColumns, schemaJson); - - mndReleaseStb(pMnode, pStb); }else{ STR_TO_VARSTR(schemaJson, "NULL"); } From 2935ebcfc4d48d3ade0737f87fa7381872a34b06 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Thu, 1 Jun 2023 17:26:03 +0800 Subject: [PATCH 195/238] set session update info --- source/libs/executor/src/timewindowoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 2cdf9bb15c..2eb8d297ac 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2904,7 +2904,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pState = pAggSup->pState; - if ((!pScanInfo->igCheckUpdate || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) && !pScanInfo->pUpdateInfo) { + if (!pScanInfo->pUpdateInfo) { pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark); } pScanInfo->twAggSup = *pTwSup; From 1036259a7b707a48902b02cb56a563c432ca37d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jun 2023 18:25:05 +0800 Subject: [PATCH 196/238] fix(query): fix error in fill. --- source/common/src/ttime.c | 1 + source/libs/executor/src/filloperator.c | 87 ++++++++++++++++++------- tests/script/tsim/parser/function.sim | 3 + 3 files changed, 69 insertions(+), 22 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 0aeb420b67..d8c43747f7 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -738,6 +738,7 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char ekey = skey; skey = tmp; } + if (unit != 'n' && unit != 'y') { return (int32_t)((ekey - skey) / interval); } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 3d1df6482e..e257879a04 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -61,6 +61,7 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); @@ -72,14 +73,16 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; getTableScanInfo(pOperator, &order, &scanFlag, false); - - int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); blockDataCleanup(pInfo->pRes); doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey); + revisedFillStartKey(pInfo, pInfo->existNewGroupBlock); + + int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows; @@ -119,6 +122,55 @@ void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int pInfo->pRes->info.id.groupId = pBlock->info.id.groupId; } +// todo refactor: decide the start key according to the query time range. +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { + int32_t order = pInfo->pFillInfo->order; + + if (order == TSDB_ORDER_ASC) { + int64_t skey = pBlock->info.window.skey; + if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); + } else if (pInfo->pFillInfo->start < skey) { + int64_t t = skey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev < pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } else { + int64_t ekey = pBlock->info.window.ekey; + if (ekey > pInfo->pFillInfo->start) { + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); + } else if (ekey < pInfo->pFillInfo->start) { + int64_t t = ekey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev < pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } +} + static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -164,28 +216,16 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); - if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) { + if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { + if (pInfo->curGroupId == 0) { + revisedFillStartKey(pInfo, pBlock); + } + pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block pInfo->totalInputRows += pInfo->pRes->info.rows; - if (order == TSDB_ORDER_ASC) { - int64_t skey = pBlock->info.window.skey; - if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the - ASSERT( taosFillNotStarted(pInfo->pFillInfo)); - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); - } - - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); - } else { - int64_t ekey = pBlock->info.window.ekey; - if (ekey > pInfo->pFillInfo->start) { - ASSERT( taosFillNotStarted(pInfo->pFillInfo)); - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); - } - - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); - } - + int64_t ts = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block pInfo->existNewGroupBlock = pBlock; @@ -276,6 +316,9 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; + +// STimeWindow w = {0}; +// getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 7f69aa2d02..120c4b8148 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -954,11 +954,14 @@ endi print =========================>TD-5190 sql select _wstart, stddev(f1) from st1 where ts>'2021-07-01 1:1:1' and ts<'2021-07-30 00:00:00' interval(1d) fill(NULL); if $rows != 29 then + print expect 29, actual: $rows return -1 endi + if $data00 != @21-07-01 00:00:00.000@ then return -1 endi + if $data01 != NULL then return -1 endi From 56782a5d41eca1b85bf66ec1c48362188c351c43 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 1 Jun 2023 18:29:16 +0800 Subject: [PATCH 197/238] fix:open task case & modify mqRebVgReq encode/decode style --- include/common/tmsg.h | 60 +++++++++++++--------- source/dnode/mnode/impl/src/mndSubscribe.c | 19 +++++-- source/dnode/vnode/src/tq/tq.c | 19 ++++--- tests/parallel_test/cases.task | 2 +- 4 files changed, 65 insertions(+), 35 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b4674aba54..d78e771fcf 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2807,39 +2807,49 @@ typedef struct { int64_t suid; } SMqRebVgReq; -static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->leftForVer); - tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); - tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); - tlen += taosEncodeString(buf, pReq->subKey); - tlen += taosEncodeFixedI8(buf, pReq->subType); - tlen += taosEncodeFixedI8(buf, pReq->withMeta); +static FORCE_INLINE int tEncodeSMqRebVgReq(SEncoder *pCoder, const SMqRebVgReq* pReq) { + if (tStartEncode(pCoder) < 0) return -1; + if (tEncodeI64(pCoder, pReq->leftForVer) < 0) return -1; + if (tEncodeI32(pCoder, pReq->vgId) < 0) return -1; + if (tEncodeI64(pCoder, pReq->oldConsumerId) < 0) return -1; + if (tEncodeI64(pCoder, pReq->newConsumerId) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->subKey) < 0) return -1; + if (tEncodeI8(pCoder, pReq->subType) < 0) return -1; + if (tEncodeI8(pCoder, pReq->withMeta) < 0) return -1; + if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { - tlen += taosEncodeString(buf, pReq->qmsg); + if (tEncodeCStr(pCoder, pReq->qmsg) < 0) return -1; } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { - tlen += taosEncodeFixedI64(buf, pReq->suid); - tlen += taosEncodeString(buf, pReq->qmsg); + if (tEncodeI64(pCoder, pReq->suid) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->qmsg) < 0) return -1; } - return tlen; + tEndEncode(pCoder); + return 0; } -static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->leftForVer); - buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); - buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); - buf = taosDecodeStringTo(buf, pReq->subKey); - buf = taosDecodeFixedI8(buf, &pReq->subType); - buf = taosDecodeFixedI8(buf, &pReq->withMeta); +static FORCE_INLINE int tDecodeSMqRebVgReq(SDecoder *pCoder, SMqRebVgReq* pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->leftForVer) < 0) return -1; + + if (tDecodeI32(pCoder, &pReq->vgId) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->oldConsumerId) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->newConsumerId) < 0) return -1; + if (tDecodeCStrTo(pCoder, pReq->subKey) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->subType) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->withMeta) < 0) return -1; + if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { - buf = taosDecodeString(buf, &pReq->qmsg); + if (tDecodeCStr(pCoder, &pReq->qmsg) < 0) return -1; } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { - buf = taosDecodeFixedI64(buf, &pReq->suid); - buf = taosDecodeString(buf, &pReq->qmsg); + if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; + if (!tDecodeIsEnd(pCoder)){ + if (tDecodeCStr(pCoder, &pReq->qmsg) < 0) return -1; + } } - return (void*)buf; + + tEndDecode(pCoder); + return 0; } typedef struct { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e62102fa77..74421afa33 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -111,7 +111,14 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri req.suid = pSub->stbUid; tstrncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); - int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); + int32_t tlen = 0; + int32_t ret = 0; + tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret); + if (ret < 0) { + return -1; + } + + tlen += sizeof(SMsgHead); void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -123,8 +130,14 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri pMsgHead->contLen = htonl(tlen); pMsgHead->vgId = htonl(pRebVg->pVgEp->vgId); - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqRebVgReq(&abuf, &req); + SEncoder encoder = {0}; + tEncoderInit(&encoder, POINTER_SHIFT(buf, sizeof(SMsgHead)), tlen); + if (tEncodeSMqRebVgReq(&encoder, &req) < 0) { + taosMemoryFreeClear(buf); + tEncoderClear(&encoder); + return -1; + } + tEncoderClear(&encoder); *pBuf = buf; *pLen = tlen; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 11760b5fd0..35055a00e3 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -633,7 +633,16 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { int ret = 0; SMqRebVgReq req = {0}; - tDecodeSMqRebVgReq(msg, &req); + SDecoder dc = {0}; + + tDecoderInit(&dc, msg, msgLen); + + // decode req + if (tDecodeSMqRebVgReq(&dc, &req) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&dc); + return -1; + } SVnode* pVnode = pTq->pVnode; int32_t vgId = TD_VID(pVnode); @@ -680,8 +689,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - pHandle->execHandle.execCol.qmsg = req.qmsg; - req.qmsg = NULL; + pHandle->execHandle.execCol.qmsg = taosStrdup(req.qmsg);; pHandle->execHandle.task = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, &pHandle->execHandle.numOfCols, req.newConsumerId); @@ -701,8 +709,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); pHandle->execHandle.execTb.suid = req.suid; - pHandle->execHandle.execTb.qmsg = req.qmsg; - req.qmsg = NULL; + pHandle->execHandle.execTb.qmsg = taosStrdup(req.qmsg); if(strcmp(pHandle->execHandle.execTb.qmsg, "") != 0) { if (nodesStringToNode(pHandle->execHandle.execTb.qmsg, &pHandle->execHandle.execTb.node) != 0) { @@ -766,7 +773,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg end: taosWUnLockLatch(&pTq->lock); - taosMemoryFree(req.qmsg); + tDecoderClear(&dc); return ret; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6a9a2b609a..0b235d0dc8 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -564,7 +564,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel.py -#,,n,system-test,python3 ./test.py -f 0-others/compatibility.py +,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py From 0315e895918de6cad123a1e136c45dbbbbe38402 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jun 2023 11:13:01 +0000 Subject: [PATCH 198/238] Avoid creating the same ID task multiple times --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- source/libs/stream/src/streamMeta.c | 48 ++++++++++--------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index eabec058d7..bd72566e5a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -719,7 +719,7 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t qDebug("succ to open rocksdb cf"); } // close default cf - rocksdb_column_family_handle_destroy(cfHandle[0]); + if (((rocksdb_column_family_handle_t**)cfHandle)[0] != 0) rocksdb_column_family_handle_destroy(cfHandle[0]); rocksdb_options_destroy(cfOpts[0]); handle->db = db; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 98e63f7f51..e3f8b12367 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -205,24 +205,25 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { // add to the ready tasks hash map, not the restored tasks hash map int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { - if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { - tFreeStreamTask(pTask); - return -1; - } - - if (streamMetaSaveTask(pMeta, pTask) < 0) { - tFreeStreamTask(pTask); - return -1; - } - - if (streamMetaCommit(pMeta) < 0) { - tFreeStreamTask(pTask); - return -1; - } - void* p = taosHashGet(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); if (p == NULL) { + if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { + tFreeStreamTask(pTask); + return -1; + } + + if (streamMetaSaveTask(pMeta, pTask) < 0) { + tFreeStreamTask(pTask); + return -1; + } + + if (streamMetaCommit(pMeta) < 0) { + tFreeStreamTask(pTask); + return -1; + } taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); + } else { + return 0; } taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES); @@ -359,18 +360,19 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { tDecodeStreamTask(&decoder, pTask); tDecoderClear(&decoder); - if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCur); - return -1; - } - + // remove duplicate void* p = taosHashGet(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); if (p == NULL) { + if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + return -1; + } taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); + } else { + continue; } - if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, sizeof(void*)) < 0) { tdbFree(pKey); tdbFree(pVal); From 32c3cfd51a767baea59c756e4302c5410f36ad03 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jun 2023 11:13:27 +0000 Subject: [PATCH 199/238] Avoid creating the same ID task multiple times --- include/libs/stream/tstream.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 1d4bbf073e..8316e6ef50 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -206,7 +206,7 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { void* streamQueueNextItem(SStreamQueue* queue); SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type); -void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); +void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit); SStreamDataSubmit* streamSubmitBlockClone(SStreamDataSubmit* pSubmit); @@ -284,7 +284,7 @@ struct SStreamTask { int16_t dispatchMsgType; SStreamStatus status; int32_t selfChildId; - int32_t nodeId; // vgroup id + int32_t nodeId; // vgroup id SEpSet epSet; SCheckpointInfo chkInfo; STaskExec exec; @@ -346,12 +346,14 @@ typedef struct SStreamMeta { void* streamBackend; int32_t streamBackendId; int64_t streamBackendRid; + SHashObj* pTaskBackendUnique; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); -SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, SArray* pTaskList); +SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHistory, int64_t triggerParam, + SArray* pTaskList); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); void tFreeStreamTask(SStreamTask* pTask); From 1571ea844af5bdc52407326b37d081b1c9d55d24 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 1 Jun 2023 11:25:55 +0000 Subject: [PATCH 200/238] Avoid creating the same ID task multiple times --- source/libs/stream/src/streamMeta.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index e3f8b12367..8c26052fdb 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -367,16 +367,22 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); + taosMemoryFree(pTask); return -1; } taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); } else { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCur); + taosMemoryFree(pTask); continue; } if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, sizeof(void*)) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); + taosMemoryFree(pTask); return -1; } From 403617976c500b77066519e752e6dffbff7b8e48 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jun 2023 22:45:03 +0800 Subject: [PATCH 201/238] fix(query): fix error in fill. --- source/libs/executor/src/filloperator.c | 40 ++++++++++++------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index e257879a04..75bfe7b671 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -61,24 +61,24 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; -static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock); +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; SSDataBlock* pResBlock = pInfo->pFinalRes; - int32_t order = TSDB_ORDER_ASC; +// int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag, false); +// getTableScanInfo(pOperator, &order, &scanFlag, false); taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); blockDataCleanup(pInfo->pRes); doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); - revisedFillStartKey(pInfo, pInfo->existNewGroupBlock); + revisedFillStartKey(pInfo, pInfo->existNewGroupBlock, order); int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey; taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); @@ -93,7 +93,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp } static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows; taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows); @@ -103,7 +103,7 @@ static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOpera // handle the cached new group data block if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); } } @@ -123,9 +123,7 @@ void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int } // todo refactor: decide the start key according to the query time range. -static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { - int32_t order = pInfo->pFillInfo->order; - +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order) { if (order == TSDB_ORDER_ASC) { int64_t skey = pBlock->info.window.skey; if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the @@ -137,7 +135,7 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { while(1) { int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - if (prev < pInfo->pFillInfo->start) { + if (prev <= pInfo->pFillInfo->start) { t = prev; break; } @@ -158,7 +156,7 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { while(1) { int64_t prev = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - if (prev < pInfo->pFillInfo->start) { + if (prev >= pInfo->pFillInfo->start) { t = prev; break; } @@ -184,12 +182,6 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { int32_t scanFlag = MAIN_SCAN; getTableScanInfo(pOperator, &order, &scanFlag, false); - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); - if (pResBlock->info.rows > 0) { - pResBlock->info.id.groupId = pInfo->curGroupId; - return pResBlock; - } - SOperatorInfo* pDownstream = pOperator->pDownstream[0]; // the scan order may be different from the output result order for agg interval operator. @@ -197,6 +189,12 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { order = ((SIntervalAggOperatorInfo*) pDownstream->info)->resultTsOrder; } + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); + if (pResBlock->info.rows > 0) { + pResBlock->info.id.groupId = pInfo->curGroupId; + return pResBlock; + } + while (1) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { @@ -218,7 +216,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { if (pInfo->curGroupId == 0) { - revisedFillStartKey(pInfo, pBlock); + revisedFillStartKey(pInfo, pBlock, order); } pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block @@ -249,7 +247,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { return pResBlock; } - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; @@ -257,7 +255,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { } else if (pInfo->existNewGroupBlock) { // try next group blockDataCleanup(pResBlock); - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows > pResultInfo->threshold) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; From bbf3f1ccd92af410c0f50bcb6487693cb862ea38 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 00:22:35 +0800 Subject: [PATCH 202/238] fix(query): add more check. --- source/libs/executor/src/filloperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 75bfe7b671..e2862b19c7 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -215,7 +215,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { - if (pInfo->curGroupId == 0) { + if (pInfo->curGroupId == 0 && taosFillNotStarted(pInfo->pFillInfo)) { revisedFillStartKey(pInfo, pBlock, order); } From 420ae73728044b13fa51743adebfba303240e480 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 00:35:45 +0000 Subject: [PATCH 203/238] Avoid creating the same ID task multiple times --- source/libs/stream/src/streamBackendRocksdb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index bd72566e5a..c743ecf7f7 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -128,7 +128,9 @@ void* streamBackendInit(const char* path) { */ streamStateOpenBackendCf(pHandle, (char*)path, cfs, nCf); } - rocksdb_list_column_families_destroy(cfs, nCf); + if (cfs != NULL) { + rocksdb_list_column_families_destroy(cfs, nCf); + } return (void*)pHandle; _EXIT: From 274771d7b4cb5329ba1267633e99d527b5a42846 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 2 Jun 2023 08:59:00 +0800 Subject: [PATCH 204/238] TD-24565 --- source/libs/executor/src/scanoperator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 28abee4342..39907be6d2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1072,6 +1072,7 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t ver) { pTableScanInfo->base.cond.twindows = *pWin; + pTableScanInfo->base.cond.startVersion = 0; pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; @@ -1194,6 +1195,7 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ } STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; + qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, pInfo->pUpdateInfo->maxDataVersion); resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; return true; @@ -1889,6 +1891,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } else { + pInfo->pUpdateInfo->maxDataVersion = pTaskInfo->streamInfo.fillHistoryVer2; doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); } } From becc5c51a40c1d1e9f14b65f54eef9e3f939f33a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 02:35:15 +0000 Subject: [PATCH 205/238] fix compatibility --- source/libs/stream/src/streamBackendRocksdb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c8a6597bad..c54f055e6a 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1924,17 +1924,17 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) { int code = 0; - STREAM_STATE_PUT_ROCKSDB(pState, "default", &key, pVal, pVLen); + STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) { int code = 0; - STREAM_STATE_GET_ROCKSDB(pState, "default", &key, pVal, pVLen); + STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen); return code; } int32_t streamDefaultDel_rocksdb(SStreamState* pState, const void* key) { int code = 0; - STREAM_STATE_DEL_ROCKSDB(pState, "default", &key); + STREAM_STATE_DEL_ROCKSDB(pState, "default", key); return code; } From 27f791ac7fc071bc7899d0a0188137686529eaf0 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 2 Jun 2023 11:04:02 +0800 Subject: [PATCH 206/238] reset ud table name --- source/dnode/vnode/src/tq/tqSink.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6e02d5e21b..db1b5ed902 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -294,6 +294,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d char* ctbName = pDataBlock->info.parTbName; if (!ctbName[0]) { + memset(ctbName, 0, TSDB_TABLE_NAME_LEN); if (res == TSDB_CODE_SUCCESS) { memcpy(ctbName, pTableSinkInfo->tbName, strlen(pTableSinkInfo->tbName)); } else { From 5c64a730a9668f2768a13c9255f4a73e6b087046 Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 11:17:38 +0800 Subject: [PATCH 207/238] enchance: change goes as static link --- cmake/cmake.options | 2 +- cmake/cmake.platform | 10 ---------- packaging/tools/makepkg.sh | 3 +++ 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index b00ae14715..555b72cbdf 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -80,7 +80,7 @@ ENDIF () option( BUILD_GEOS "If build geos on Windows" - OFF + ON ) option( diff --git a/cmake/cmake.platform b/cmake/cmake.platform index f9faf7316c..ba747c6134 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -56,17 +56,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(TD_DARWIN TRUE) SET(OSTYPE "macOS") - execute_process(COMMAND geos-config --cflags OUTPUT_VARIABLE GEOS_CFLAGS) - execute_process(COMMAND geos-config --ldflags OUTPUT_VARIABLE GEOS_LDFLAGS) - string(SUBSTRING ${GEOS_CFLAGS} 2 -1 GEOS_CFLAGS) - string(REGEX REPLACE "\n" "" GEOS_CFLAGS ${GEOS_CFLAGS}) - string(SUBSTRING ${GEOS_LDFLAGS} 2 -1 GEOS_LDFLAGS) - string(REGEX REPLACE "\n" "" GEOS_LDFLAGS ${GEOS_LDFLAGS}) - MESSAGE("GEOS_CFLAGS "${GEOS_CFLAGS}) - MESSAGE("GEOS_LDFLAGS "${GEOS_LDFLAGS}) ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") - INCLUDE_DIRECTORIES(${GEOS_CFLAGS}) - LINK_DIRECTORIES(${GEOS_LDFLAGS}) IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") MESSAGE("Current system arch is arm64") diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 2f1e803689..7cfd9aa664 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -112,9 +112,11 @@ fi if [ "$osType" == "Darwin" ]; then lib_files="${build_dir}/lib/libtaos.${version}.dylib" wslib_files="${build_dir}/lib/libtaosws.dylib" + rocksdb_lib_files="${build_dir}/lib/librocksdb.dylib.8.1.1" else lib_files="${build_dir}/lib/libtaos.so.${version}" wslib_files="${build_dir}/lib/libtaosws.so" + rocksdb_lib_files="${build_dir}/lib/librocksdb.so.8.1.1" fi header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" @@ -337,6 +339,7 @@ fi # Copy driver mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : +[ -f ${rocksdb_lib_files} ] && cp ${rocksdb_lib_files} ${install_dir}/driver || : # Copy connector if [ "$verMode" == "cluster" ]; then From 27b7d1ec8818672d93e6e0d5c47c084b0b140ae1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 11:17:44 +0800 Subject: [PATCH 208/238] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/executor/src/scanoperator.c | 4 ++-- source/libs/stream/src/stream.c | 10 +++++----- source/libs/stream/src/streamDispatch.c | 6 +++--- source/libs/stream/src/streamExec.c | 12 ++++++------ 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8cc791d1a6..5e7e21e7bd 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1086,7 +1086,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - qDebug("s-task:%s set the start wal offset to be:%"PRId64, pTask->id.idStr, sversion); + qDebug("s-task:%s set start wal scan start ver:%"PRId64, pTask->id.idStr, sversion); walReaderSeekVer(pTask->exec.pWalReader, sversion); pTask->chkInfo.currentVer = sversion; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 88f5642ef9..80a7505c77 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2005,7 +2005,7 @@ FETCH_NEXT_BLOCK: // printDataBlock(pBlock, "stream scan recv"); return pBlock; } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) { - qDebug("scan mode %d", pInfo->scanMode); + qDebug("stream scan mode:%d, %s", pInfo->scanMode, id); switch (pInfo->scanMode) { case STREAM_SCAN_FROM_RES: { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; @@ -2122,7 +2122,7 @@ FETCH_NEXT_BLOCK: pInfo->numOfExec++; pOperator->resultInfo.totalRows += pBlockInfo->rows; - qDebug("stream scan get source rows:%" PRId64", %s", pBlockInfo->rows, id); + qDebug("stream scan completed, and return source rows:%" PRId64", %s", pBlockInfo->rows, id); if (pBlockInfo->rows > 0) { return pBlock; } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index acc69c5a2b..7457b2197e 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -296,11 +296,8 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { if (type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* px = (SStreamDataSubmit*)pItem; - qDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr, - px->submit.msgLen, px->submit.ver, total, size); - if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && tInputQueueIsFull(pTask)) { - qError("s-task:%s input queue is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) abort", + qError("s-task:%s input queue is full, capacity(size:%d num:%dMiB), current(blocks:%d, size:%.2fMiB) stop to push data", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); streamDataSubmitDestroy(px); @@ -314,9 +311,12 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { taosFreeQitem(pItem); return code; } + + qDebug("s-task:%s submit enqueue msgLen:%d ver:%" PRId64 ", total in queue:%d, size:%.2fMiB", pTask->id.idStr, + px->submit.msgLen, px->submit.ver, total, size + px->submit.msgLen/1048576.0); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || type == STREAM_INPUT__REF_DATA_BLOCK) { - if (/*(pTask->taskLevel == TASK_LEVEL__SOURCE) && */(tInputQueueIsFull(pTask))) { + if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && (tInputQueueIsFull(pTask))) { qError("s-task:%s input queue is full, capacity:%d size:%d MiB, current(blocks:%d, size:%.2fMiB) abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY, STREAM_TASK_INPUT_QUEUEU_CAPACITY_IN_SIZE, total, size); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 1e939cb071..bd6a013de2 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -283,7 +283,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov msg.info.noResp = 1; tmsgSendReq(pEpSet, &msg); - qDebug("s-task:%s dispatch recover finish msg to taskId:%d node %d: recover finish msg", pTask->id.idStr, + qDebug("s-task:%s dispatch recover finish msg to downstream taskId:0x%x node %d: recover finish msg", pTask->id.idStr, pReq->taskId, vgId); return 0; @@ -414,7 +414,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat req.taskId = downstreamTaskId; - qDebug("s-task:%s (child taskId:%d) fix-dispatch blocks:%d to down stream s-task:%d in vgId:%d", pTask->id.idStr, + qDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to down stream s-task:0x%x in vgId:%d", pTask->id.idStr, pTask->selfChildId, numOfBlocks, downstreamTaskId, vgId); code = doSendDispatchMsg(pTask, &req, vgId, pEpSet); @@ -514,7 +514,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { return 0; } - qDebug("s-task:%s start to dispatch msg, output status:%d", pTask->id.idStr, pTask->outputStatus); + qDebug("s-task:%s start to dispatch msg, set output status:%d", pTask->id.idStr, pTask->outputStatus); SStreamDataBlock* pDispatchedBlock = streamQueueNextItem(pTask->outputQueue); if (pDispatchedBlock == NULL) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index c3dd848bc7..813bc50b87 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -236,11 +236,11 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { taosFreeQitem(qRes); return code; } - - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - qDebug("s-task:%s scan exec dispatch blocks:%d", pTask->id.idStr, batchCnt); - streamDispatchStreamBlock(pTask); - } +// +// if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { +// qDebug("s-task:%s scan exec dispatch blocks:%d", pTask->id.idStr, batchCnt); +// streamDispatchStreamBlock(pTask); +// } if (finished) { break; @@ -438,7 +438,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { SArray* pBlockList = pMerged->submits; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s %p set submit input (merged), batch num:%d", id, pTask, numOfBlocks); + qDebug("s-task:%s %p set (merged) submit blocks as a batch, numOfBlocks:%d", id, pTask, numOfBlocks); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)pInput; From 63d9a5dcf37220262e11db2c3a96c2626fa798d7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 04:08:08 +0000 Subject: [PATCH 209/238] avoid mem leak --- source/libs/stream/src/tstreamFileState.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index f531f65565..54832b042a 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -16,12 +16,12 @@ #include "tstreamFileState.h" #include "query.h" +#include "storageapi.h" #include "streamBackendRocksdb.h" #include "taos.h" #include "tcommon.h" #include "thash.h" #include "tsimplehash.h" -#include "storageapi.h" #define FLUSH_RATIO 0.5 #define FLUSH_NUM 4 @@ -420,6 +420,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { return TSDB_CODE_FAILED; } sscanf(val, "%" PRId64 "", &maxCheckPointId); + taosMemoryFree(val); } for (int64_t i = maxCheckPointId; i > 0; i--) { char buf[128] = {0}; @@ -435,9 +436,11 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (ts < mark) { // statekey winkey.ts < mark forceRemoveCheckpoint(pFileState, i); + taosMemoryFreeClear(val); break; } else { } + taosMemoryFree(val); } return code; } From f1bd829977b43a0bbb12b4f9acdfb13d57f4a87d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 13:10:54 +0800 Subject: [PATCH 210/238] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tq/tq.c | 3 ++- source/dnode/vnode/src/tq/tqMeta.c | 3 ++- source/dnode/vnode/src/tq/tqRead.c | 19 +++++++++++-------- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/scanoperator.c | 5 +++-- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamExec.c | 4 ++-- 8 files changed, 23 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index fbe2f4809c..8003419ec9 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -200,7 +200,7 @@ STqReader *tqReaderOpen(SVnode *pVnode); void tqReaderClose(STqReader *); void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); -int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList, const char* id); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5e7e21e7bd..a6bed26884 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -708,8 +708,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } + pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); - tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); + tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList, NULL); taosArrayDestroy(tbUidList); buildSnapContext(handle.vnode, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index be5b5706e2..a720e556e6 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -344,8 +344,9 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } + handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); - tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); + tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList, NULL); taosArrayDestroy(tbUidList); buildSnapContext(reader.vnode, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index dbb7d564f8..f15a9898b5 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -394,8 +394,8 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) { SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pReader->tbIdHash == NULL) { - SSDataBlock* pRes = NULL; - int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); + SSDataBlock* pRes = NULL; + int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { return true; } @@ -457,7 +457,7 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { - tqDebug("tq reader next data block, len:%d ver:%" PRId64 " index:%d/%d, %s", pReader->msg.msgLen, pReader->msg.ver, + tqDebug("try next data block, len:%d ver:%" PRId64 " index:%d/%d, %s", pReader->msg.msgLen, pReader->msg.ver, pReader->nextBlk, numOfBlocks, idstr); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); @@ -467,10 +467,11 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); if (ret != NULL) { - tqDebug("tq reader block found, ver:%" PRId64 ", uid:%" PRId64, pReader->msg.ver, pSubmitTbData->uid); + tqDebug("block found, ver:%" PRId64 ", uid:%" PRId64", %s", pReader->msg.ver, pSubmitTbData->uid, idstr); return true; } else { - tqDebug("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid); + tqDebug("discard submit block, uid:%" PRId64 ", total queried tables:%d continue %s", pSubmitTbData->uid, + taosHashGetSize(pReader->tbIdHash), idstr); } pReader->nextBlk++; @@ -604,7 +605,6 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) { tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); - SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); SSDataBlock* pBlock = pReader->pResBlock; @@ -612,6 +612,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* blockDataCleanup(pBlock); + int32_t vgId = pReader->pWalReader->pWal->cfg.vgId; int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; @@ -628,7 +629,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64 "version %d, possibly dropped table", - pReader->pWalReader->pWal->cfg.vgId, suid, uid, pReader->cachedSchemaVer); + vgId, suid, uid, pReader->cachedSchemaVer); pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -642,6 +643,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* if (blockDataGetNumOfCols(pBlock) == 0) { int32_t code = buildResSDataBlock(pReader->pResBlock, pReader->pSchemaWrapper, pReader->pColIdList); if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d failed to build data block, code:%s", vgId, tstrerror(code)); return code; } } @@ -998,7 +1000,7 @@ FAIL: void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } -int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { +int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList, const char* id) { if (pReader->tbIdHash) { taosHashClear(pReader->tbIdHash); } else { @@ -1015,6 +1017,7 @@ int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } + tqDebug("s-task:%s %d tables are set to be queried target table", id, (int32_t) taosArrayGetSize(tbUidList)); return 0; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ed8a4f3a3b..4d02362904 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -401,7 +401,7 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI int32_t code = 0; if (isAdd) { - qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id); + qDebug("try to add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id); } // traverse to the stream scanner node to add this table id diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 80a7505c77..5f09b0c67a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2308,7 +2308,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SArray* pColIds = NULL; SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + const char* idstr = pTaskInfo->id.str; if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2419,7 +2420,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys // set the extract column id to streamHandle pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds); SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); - code = pAPI->tqReaderFn.tqReaderSetQueryTableList(pInfo->tqReader, tableIdList); + code = pAPI->tqReaderFn.tqReaderSetQueryTableList(pInfo->tqReader, tableIdList, idstr); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index bd6a013de2..9501718416 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -318,7 +318,7 @@ int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, in msg.pCont = buf; msg.msgType = pTask->dispatchMsgType; - qDebug("dispatch from s-task:%s to taskId:0x%x vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); + qDebug("s-task:%s dispatch msg to taskId:0x%x vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); tmsgSendReq(pEpSet, &msg); code = 0; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 813bc50b87..c28cd6f334 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -133,7 +133,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i taosArrayPush(pRes, &block); - qDebug("s-task:%s (child %d) executed and get block, total blocks:%d, size:%.2fMiB", pTask->id.idStr, + qDebug("s-task:%s (child %d) executed and get %d result blocks, size:%.2fMiB", pTask->id.idStr, pTask->selfChildId, numOfBlocks, size / 1048576.0); // current output should be dispatched to down stream nodes @@ -371,7 +371,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { streamQueueProcessSuccess(pTask->inputQueue); if (batchSize > MAX_STREAM_EXEC_BATCH_NUM) { - qDebug("s-task:%s maximum batch limit:%d reached, processing this batch of blocks", id, + qDebug("s-task:%s batch size limit:%d reached, start to process blocks", id, MAX_STREAM_EXEC_BATCH_NUM); break; } From dca2abd8d1fccd7688f0df6bec66db368fe59ffb Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Jun 2023 13:57:54 +0800 Subject: [PATCH 211/238] feat: change get taos-tools with 3.0 branch --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d9d2f12069..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 4378702 + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 3b69027325f59d03ed7893fdbb326e3392244c13 Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 14:01:25 +0800 Subject: [PATCH 212/238] enhance: supply librocksdb.so install and uninstall --- packaging/tools/install.sh | 9 +++++++++ packaging/tools/post.sh | 6 ++++++ packaging/tools/remove.sh | 3 +++ 3 files changed, 18 insertions(+) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 1b47b10520..9452e48d33 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -250,18 +250,27 @@ function install_lib() { # Remove links ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : + ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : #${csudo}rm -rf ${v15_java_app_dir} || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib_link_dir}/librocksdb.so.8 + ${csudo}ln -sf ${lib_link_dir}/librocksdb.so.8 ${lib_link_dir}/librocksdb.so + + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib64_link_dir}/librocksdb.so.8 || : + ${csudo}ln -sf ${lib64_link_dir}/librocksdb.so.8 ${lib64_link_dir}/librocksdb.so || : + [ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index fc392c9684..4a8dda1d30 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -202,10 +202,15 @@ function install_lib() { log_print "start install lib from ${lib_dir} to ${lib_link_dir}" ${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || : + + #rocksdb + [ -f ${lib_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb* || : + [ -f ${lib64_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib64_link_dir}/librocksdb* || : [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : + ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 @@ -214,6 +219,7 @@ function install_lib() { if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.${lib_file_ext} ]]; then ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 + ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib64_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} 2>>${install_log_path} fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 6c671473bf..a17b29983c 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -142,11 +142,14 @@ function clean_local_bin() { function clean_lib() { # Remove link ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : + } function clean_header() { From eb344891da96d8d58746707ca06eae7a3f479188 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 2 Jun 2023 14:03:53 +0800 Subject: [PATCH 213/238] feat: get taos-tools code from main branch --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d9d2f12069..9bbda8309f 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 4378702 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From f59161464d7f596418b34915d3bd51368675e50c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 06:48:26 +0000 Subject: [PATCH 214/238] avoid mem leak --- source/libs/executor/src/scanoperator.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 146552b058..6d17672513 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +// clang-format off + #include "executorInt.h" #include "filter.h" #include "function.h" @@ -2444,6 +2446,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys int32_t len = 0; pAPI->stateStore.streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_NAME, strlen(STREAM_SCAN_OP_NAME), &buff, &len); streamScanOperatorDecode(buff, len, pInfo); + taosMemoryFree(buff); } setOperatorInfo(pOperator, STREAM_SCAN_OP_NAME, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, @@ -3458,3 +3461,5 @@ static void destoryTableCountScanOperator(void* param) { taosArrayDestroy(pTableCountScanInfo->stbUidList); taosMemoryFreeClear(param); } + +// clang-format on From 76106165daf887be46463b78f49cc21a1d3c59f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 15:13:25 +0800 Subject: [PATCH 215/238] other: add some logs. --- source/libs/executor/src/scanoperator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index cea1e6361d..83a4f91034 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2090,7 +2090,9 @@ FETCH_NEXT_BLOCK: while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) { SSDataBlock* pRes = NULL; int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); + if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { + qDebug("failed to retrieve data from block, code:%s, rows:%"PRId64 " try next block in submit block", tstrerror(code), pRes->info.rows); continue; } From bcd8867fb2010892bdae2b3fa92edef8b4e60fd0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 15:27:14 +0800 Subject: [PATCH 216/238] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 83a4f91034..8151c459e5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1897,6 +1897,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pCreateTbRes, "recover createTbl"); return pInfo->pCreateTbRes; } + qDebug("stream recover scan get block, rows %" PRId64, pInfo->pRecoverRes->info.rows); printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; @@ -2089,10 +2090,13 @@ FETCH_NEXT_BLOCK: while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) { SSDataBlock* pRes = NULL; + int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); + qDebug("retrieve data from submit completed code:%s, rows:%" PRId64 " %s", + tstrerror(code), pRes->info.rows, id); if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { - qDebug("failed to retrieve data from block, code:%s, rows:%"PRId64 " try next block in submit block", tstrerror(code), pRes->info.rows); + qDebug("retrieve data failed, try next block in submit block, %s", id); continue; } @@ -2100,6 +2104,7 @@ FETCH_NEXT_BLOCK: if (pInfo->pCreateTbRes->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_RES; + qDebug("create table res exists, rows:%"PRId64" return from stream scan, %s", pInfo->pCreateTbRes->info.rows, id); return pInfo->pCreateTbRes; } From 2edba85efe930fe324c1c008799c1a7bfa8cadef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 15:27:44 +0800 Subject: [PATCH 217/238] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8151c459e5..0ed41aeea1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2092,8 +2092,8 @@ FETCH_NEXT_BLOCK: SSDataBlock* pRes = NULL; int32_t code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id); - qDebug("retrieve data from submit completed code:%s, rows:%" PRId64 " %s", - tstrerror(code), pRes->info.rows, id); + qDebug("retrieve data from submit completed code:%s, rows:%" PRId64 " %s", tstrerror(code), pRes->info.rows, + id); if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) { qDebug("retrieve data failed, try next block in submit block, %s", id); From 822f2673990766daf5d6403ecc054ea4c73e255f Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 15:36:36 +0800 Subject: [PATCH 218/238] enhance: install and uninstall rocksb.so in deb and rpm --- packaging/deb/DEBIAN/preinst | 1 + packaging/deb/DEBIAN/prerm | 1 + packaging/deb/makedeb.sh | 1 + packaging/rpm/tdengine.spec | 2 ++ 4 files changed, 5 insertions(+) diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index 904a946e20..d9951fc4d4 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -80,4 +80,5 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : +${csudo}rm -f ${install_main_dir}/driver/librocksdb.* || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 0d63115a04..80799ae0e5 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -40,6 +40,7 @@ else ${csudo}rm -f ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${log_link_dir} || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 9f49cf345a..624ccb0e3b 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -94,6 +94,7 @@ fi cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver +cp ${compile_dir}/build/lib/librocksdb.8 ${pkg_dir}${install_home_path}/driver [ -f ${compile_dir}/build/lib/${wslibfile} ] && cp ${compile_dir}/build/lib/${wslibfile} ${pkg_dir}${install_home_path}/driver ||: cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 52d5335003..31d597785e 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -174,6 +174,7 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f %{homepath}/driver/libtaos* || : +${csudo}rm -f %{homepath}/driver/librocksdb* || : #Scripts executed after installation %post @@ -219,6 +220,7 @@ if [ $1 -eq 0 ];then ${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : ${csudo}rm -f ${log_link_dir} || : ${csudo}rm -f ${data_link_dir} || : From 3d7defa881d8f8a146ffa6cba663d2ffe9a3677f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 15:43:38 +0800 Subject: [PATCH 219/238] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0ed41aeea1..df2062afe6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2113,6 +2113,8 @@ FETCH_NEXT_BLOCK: pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); + qDebug("%" PRId64 " rows in datablock, update res:%" PRId64 " %s", pBlockInfo->rows, + pInfo->pUpdateDataRes->info.rows, id); if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { break; } From 53696caabf16b91c1e621d6d1e11f1083fd1f9df Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 16:52:36 +0800 Subject: [PATCH 220/238] fix: rpm and deb librocksdb.so install and unintall --- packaging/deb/DEBIAN/preinst | 2 +- packaging/deb/DEBIAN/prerm | 2 +- packaging/deb/makedeb.sh | 3 ++- packaging/rpm/tdengine.spec | 2 ++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index d9951fc4d4..d6558d5b3b 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -80,5 +80,5 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : -${csudo}rm -f ${install_main_dir}/driver/librocksdb.* || : +[ -f ${install_main_dir}/driver/librocksdb.* ] && ${csudo}rm -f ${install_main_dir}/driver/librocksdb.* || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 80799ae0e5..8f8d472867 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -40,7 +40,7 @@ else ${csudo}rm -f ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : + [ -f ${lib_link_dir}/librocksdb.* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${log_link_dir} || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 624ccb0e3b..024c69deb1 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -31,6 +31,7 @@ cd ${pkg_dir} libfile="libtaos.so.${tdengine_ver}" wslibfile="libtaosws.so" +rocksdblib="librocksdb.so.8" # create install dir install_home_path="/usr/local/taos" @@ -94,7 +95,7 @@ fi cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver -cp ${compile_dir}/build/lib/librocksdb.8 ${pkg_dir}${install_home_path}/driver +[ -f ${compile_dir}/build/lib/${rocksdblib} ] && cp ${compile_dir}/build/lib/${rocksdblib} ${pkg_dir}${install_home_path}/driver ||: [ -f ${compile_dir}/build/lib/${wslibfile} ] && cp ${compile_dir}/build/lib/${wslibfile} ${pkg_dir}${install_home_path}/driver ||: cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 31d597785e..2b056c376a 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -45,6 +45,7 @@ echo buildroot: %{buildroot} libfile="libtaos.so.%{_version}" wslibfile="libtaosws.so" +rocksdblib="librocksdb.so.8" # create install path, and cp file mkdir -p %{buildroot}%{homepath}/bin @@ -92,6 +93,7 @@ if [ -f %{_compiledir}/build/bin/taosadapter ]; then fi cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver [ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile} %{buildroot}%{homepath}/driver ||: +[ -f %{_compiledir}/build/lib/${rocksdblib} ] && cp %{_compiledir}/build/lib/${rocksdblib} %{buildroot}%{homepath}/driver ||: cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include From 9f8f5374aaecd3f07f461240e2bc42ce35a56a8f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 16:56:20 +0800 Subject: [PATCH 221/238] fix: num_of_records_per_req item in json config need less than 32768. current = 100000 --- tests/system-test/1-insert/manyVgroups.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index 3b0fa96b08..8c6f39cf96 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -11,7 +11,7 @@ "confirm_parameter_prompt": "no", "insert_interval": 0, "interlace_rows": 0, - "num_of_records_per_req": 100000, + "num_of_records_per_req": 10000, "databases": [ { "dbinfo": { @@ -73,4 +73,4 @@ ] } ] -} \ No newline at end of file +} From 27d8870e182a6f31a4590869f3a7ec3166e29fd2 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Fri, 2 Jun 2023 17:09:03 +0800 Subject: [PATCH 222/238] fix: no data after seek --- source/client/src/clientTmq.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ccb09e3584..e1b2b9c48b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1868,7 +1868,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { // update the local offset value only for the returned values, only when the local offset is NOT updated // by tmq_offset_seek function if (!pVg->seekUpdated) { + tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId); pVg->offsetInfo.currentOffset = pDataRsp->rspOffset; + } else { + tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId); } // update the status @@ -1952,8 +1955,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { return NULL; } - if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate - pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; + // update the local offset value only for the returned values, only when the local offset is NOT updated + // by tmq_offset_seek function + if (!pVg->seekUpdated) { + if(pollRspWrapper->taosxRsp.rspOffset.type != 0) { // if offset is validate + tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId); + pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; + } + } else { + tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId); } atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); From 163f5abc4ce1841a6b5f0dcde8a9505c22178af0 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Fri, 2 Jun 2023 17:22:49 +0800 Subject: [PATCH 223/238] fix: no data after seek --- source/client/src/clientTmq.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ccb09e3584..e1b2b9c48b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1868,7 +1868,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { // update the local offset value only for the returned values, only when the local offset is NOT updated // by tmq_offset_seek function if (!pVg->seekUpdated) { + tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId); pVg->offsetInfo.currentOffset = pDataRsp->rspOffset; + } else { + tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId); } // update the status @@ -1952,8 +1955,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { return NULL; } - if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate - pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; + // update the local offset value only for the returned values, only when the local offset is NOT updated + // by tmq_offset_seek function + if (!pVg->seekUpdated) { + if(pollRspWrapper->taosxRsp.rspOffset.type != 0) { // if offset is validate + tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", tmq->consumerId); + pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; + } + } else { + tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", tmq->consumerId); } atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); From 449051cda67aff90ccb2e690b5521025803b78b0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 10:05:32 +0000 Subject: [PATCH 224/238] avoid mem leak --- source/libs/stream/src/streamBackendRocksdb.c | 4 ++-- source/libs/stream/src/tstreamFileState.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index b8af05c86d..30a7cefc9c 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1056,7 +1056,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa rocksdb_readoptions_t* opts = pState->pTdbState->readOpts; \ size_t len = 0; \ char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL) { \ + if (val == NULL || len == 0) { \ if (err == NULL) { \ qTrace("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ funcname); \ @@ -1069,7 +1069,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa } else { \ char* p = NULL; \ int32_t len = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (len < 0) { \ + if (len <= 0) { \ qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ funcname); \ code = -1; \ diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 54832b042a..7ea1a41925 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -416,7 +416,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { int32_t len = 0; memcpy(buf, taskKey, strlen(taskKey)); code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len); - if (code != 0) { + if (code != 0 || len == 0 || val == NULL) { return TSDB_CODE_FAILED; } sscanf(val, "%" PRId64 "", &maxCheckPointId); From 8d3b3dbedc360a8d62bd08ff3a15d74e62035a3e Mon Sep 17 00:00:00 2001 From: xleili Date: Fri, 2 Jun 2023 18:11:56 +0800 Subject: [PATCH 225/238] enchance: main geos use static link, remove and install librocksdb --- cmake/cmake.options | 2 +- cmake/cmake.platform | 10 ---------- packaging/deb/DEBIAN/preinst | 1 + packaging/deb/DEBIAN/prerm | 1 + packaging/deb/makedeb.sh | 2 ++ packaging/rpm/tdengine.spec | 5 ++++- packaging/tools/install.sh | 9 +++++++++ packaging/tools/makepkg.sh | 3 +++ packaging/tools/post.sh | 6 ++++++ packaging/tools/remove.sh | 3 +++ 10 files changed, 30 insertions(+), 12 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index b00ae14715..555b72cbdf 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -80,7 +80,7 @@ ENDIF () option( BUILD_GEOS "If build geos on Windows" - OFF + ON ) option( diff --git a/cmake/cmake.platform b/cmake/cmake.platform index f9faf7316c..ba747c6134 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -56,17 +56,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(TD_DARWIN TRUE) SET(OSTYPE "macOS") - execute_process(COMMAND geos-config --cflags OUTPUT_VARIABLE GEOS_CFLAGS) - execute_process(COMMAND geos-config --ldflags OUTPUT_VARIABLE GEOS_LDFLAGS) - string(SUBSTRING ${GEOS_CFLAGS} 2 -1 GEOS_CFLAGS) - string(REGEX REPLACE "\n" "" GEOS_CFLAGS ${GEOS_CFLAGS}) - string(SUBSTRING ${GEOS_LDFLAGS} 2 -1 GEOS_LDFLAGS) - string(REGEX REPLACE "\n" "" GEOS_LDFLAGS ${GEOS_LDFLAGS}) - MESSAGE("GEOS_CFLAGS "${GEOS_CFLAGS}) - MESSAGE("GEOS_LDFLAGS "${GEOS_LDFLAGS}) ADD_DEFINITIONS("-DDARWIN -Wno-tautological-pointer-compare") - INCLUDE_DIRECTORIES(${GEOS_CFLAGS}) - LINK_DIRECTORIES(${GEOS_LDFLAGS}) IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64") MESSAGE("Current system arch is arm64") diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index 904a946e20..f2f98952db 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -80,4 +80,5 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : +[ -f ${install_main_dir}/driver/librocksdb.* ] && ${csudo}rm -f ${install_main_dir}/driver/librocksdb.* || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 0d63115a04..7cee258606 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -40,6 +40,7 @@ else ${csudo}rm -f ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + [ -f ${lib_link_dir}/librocksdb.* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${log_link_dir} || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 9f49cf345a..d5dbd9e4b8 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -31,6 +31,7 @@ cd ${pkg_dir} libfile="libtaos.so.${tdengine_ver}" wslibfile="libtaosws.so" +rocksdblib="librocksdb.so.8" # create install dir install_home_path="/usr/local/taos" @@ -94,6 +95,7 @@ fi cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver +[ -f ${compile_dir}/build/lib/${rocksdblib} ] && cp ${compile_dir}/build/lib/${rocksdblib} ${pkg_dir}${install_home_path}/driver ||: [ -f ${compile_dir}/build/lib/${wslibfile} ] && cp ${compile_dir}/build/lib/${wslibfile} ${pkg_dir}${install_home_path}/driver ||: cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 52d5335003..144cb0cfb2 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -45,6 +45,7 @@ echo buildroot: %{buildroot} libfile="libtaos.so.%{_version}" wslibfile="libtaosws.so" +rocksdblib="librocksdb.so.8" # create install path, and cp file mkdir -p %{buildroot}%{homepath}/bin @@ -92,6 +93,7 @@ if [ -f %{_compiledir}/build/bin/taosadapter ]; then fi cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver [ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile} %{buildroot}%{homepath}/driver ||: +[ -f %{_compiledir}/build/lib/${rocksdblib} ] && cp %{_compiledir}/build/lib/${rocksdblib} %{buildroot}%{homepath}/driver ||: cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include @@ -174,6 +176,7 @@ fi # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f %{homepath}/driver/libtaos* || : +${csudo}rm -f %{homepath}/driver/librocksdb* || : #Scripts executed after installation %post @@ -219,7 +222,7 @@ if [ $1 -eq 0 ];then ${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : - + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : ${csudo}rm -f ${log_link_dir} || : ${csudo}rm -f ${data_link_dir} || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 1b47b10520..1501b15196 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -250,18 +250,27 @@ function install_lib() { # Remove links ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : + ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : #${csudo}rm -rf ${v15_java_app_dir} || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib_link_dir}/librocksdb.so.8 + ${csudo}ln -sf ${lib_link_dir}/librocksdb.so.8 ${lib_link_dir}/librocksdb.so + [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + ${csudo}ln -sf ${install_main_dir}/driver/librocksdb.* ${lib64_link_dir}/librocksdb.so.8 || : + ${csudo}ln -sf ${lib64_link_dir}/librocksdb.so.8 ${lib64_link_dir}/librocksdb.so || : + [ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index b0537e8bcf..bf35734da3 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -111,9 +111,11 @@ fi if [ "$osType" == "Darwin" ]; then lib_files="${build_dir}/lib/libtaos.${version}.dylib" wslib_files="${build_dir}/lib/libtaosws.dylib" + rocksdb_lib_files="${build_dir}/lib/librocksdb.so.8.1.1" else lib_files="${build_dir}/lib/libtaos.so.${version}" wslib_files="${build_dir}/lib/libtaosws.so" + rocksdb_lib_files="${build_dir}/lib/librocksdb.so.8.1.1" fi header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" @@ -336,6 +338,7 @@ fi # Copy driver mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : +[ -f ${rocksdb_lib_files} ] && cp ${rocksdb_lib_files} ${install_dir}/driver || : # Copy connector if [ "$verMode" == "cluster" ]; then diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index fc392c9684..1b5eb45f66 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -203,9 +203,14 @@ function install_lib() { ${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || : + #rocksdb + [ -f ${lib_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib_link_dir}/librocksdb* || : + [ -f ${lib64_link_dir}/librocksdb* ] && ${csudo}rm -f ${lib64_link_dir}/librocksdb* || : + [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : + ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 @@ -214,6 +219,7 @@ function install_lib() { if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.${lib_file_ext} ]]; then ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 + ${csudo}ln -s ${lib_dir}/librocksdb.* ${lib64_link_dir}/librocksdb.${lib_file_ext_1} 2>>${install_log_path} || return 1 [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} 2>>${install_log_path} fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 6c671473bf..a17b29983c 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -142,11 +142,14 @@ function clean_local_bin() { function clean_lib() { # Remove link ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/librocksdb.* || : [ -f ${lib_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/librocksdb.* || : [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : + } function clean_header() { From 17f2c3f88249b07ed79c5888bdf505c1322d9f20 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 2 Jun 2023 16:51:44 +0800 Subject: [PATCH 226/238] reset wind info --- source/libs/executor/inc/executorInt.h | 1 + source/libs/executor/src/timewindowoperator.c | 25 +++++++++++++++++++ source/libs/stream/src/tstreamFileState.c | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 79391bc1c5..ffc63a22a8 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -456,6 +456,7 @@ typedef struct SStreamIntervalOperatorInfo { SSHashObj* pUpdatedMap; int64_t dataVersion; SStateStore statestore; + bool recvGetAll; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index fc08f827a5..3a83472079 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2439,6 +2439,15 @@ static inline int winPosCmprImpl(const void* pKey1, const void* pKey2) { return 0; } +static void resetUnCloseWinInfo(SSHashObj* winMap) { + void* pIte = NULL; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) { + SRowBuffPos* pPos = *(SRowBuffPos**)pIte; + pPos->beUsed = true; + } +} + static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2472,6 +2481,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->binfo.pRes; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable); + } + setOperatorCompleted(pOperator); if (!IS_FINAL_OP(pInfo)) { clearFunctionContext(&pOperator->exprSupp); @@ -2565,6 +2579,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { + pInfo->recvGetAll = true; getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { @@ -2773,6 +2788,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, compareTs, pInfo->pState, pInfo->twAggSup.deleteMark); pInfo->dataVersion = 0; pInfo->statestore = pTaskInfo->storageAPI.stateStore; + pInfo->recvGetAll = false; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -4751,6 +4767,12 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes; } + + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable); + } + setOperatorCompleted(pOperator); if (pInfo->twAggSup.maxTs > 0 && pInfo->twAggSup.maxTs - pInfo->twAggSup.checkPointInterval > pInfo->twAggSup.checkPointTs) { @@ -4790,6 +4812,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_GET_ALL) { qDebug("===stream===single interval recv|block type STREAM_GET_ALL"); + pInfo->recvGetAll = true; getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { @@ -4960,6 +4983,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); pInfo->statestore = pTaskInfo->storageAPI.stateStore; + pInfo->recvGetAll = false; + initIntervalDownStream(downstream, pPhyNode->type, pInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index f531f65565..e13499d87f 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -137,7 +137,7 @@ void clearExpiredRowBuff(SStreamFileState* pFileState, TSKEY ts, bool all) { SListNode* pNode = NULL; while ((pNode = tdListNext(&iter)) != NULL) { SRowBuffPos* pPos = *(SRowBuffPos**)(pNode->data); - if (all || (pFileState->getTs(pPos->pKey) < ts)) { + if (all || (pFileState->getTs(pPos->pKey) < ts && !pPos->beUsed)) { ASSERT(pPos->pRowBuff != NULL); tdListAppend(pFileState->freeBuffs, &(pPos->pRowBuff)); pPos->pRowBuff = NULL; From 6ee68331b319aefe7578a0051285ad4a7069588f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 11:54:36 +0000 Subject: [PATCH 227/238] update DEFAULT_BUFF --- source/libs/stream/src/streamBackendRocksdb.c | 7 +++++-- source/libs/stream/src/streamUpdate.c | 12 ++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 30a7cefc9c..e07198f6ab 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -87,8 +87,6 @@ void* streamBackendInit(const char* path) { pHandle->cfInst = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); rocksdb_env_t* env = rocksdb_create_default_env(); // rocksdb_envoptions_create(); - rocksdb_env_set_low_priority_background_threads(env, 4); - rocksdb_env_set_high_priority_background_threads(env, 2); rocksdb_cache_t* cache = rocksdb_cache_create_lru(64 << 20); @@ -577,6 +575,11 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { int64_t now = taosGetTimestampMs(); p = taosDecodeFixedI64(p, &key.unixTimestamp); p = taosDecodeFixedI32(p, &key.len); + if (vlen != sizeof(int64_t) + sizeof(int32_t) + key.len) { + *dest = NULL; + return -1; + } + if (key.len == 0) { key.data = NULL; } else { diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index fff666ec9f..85be120dbd 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -20,9 +20,9 @@ #include "ttime.h" #define DEFAULT_FALSE_POSITIVE 0.01 -#define DEFAULT_BUCKET_SIZE 1310720 -#define DEFAULT_MAP_CAPACITY 1310720 -#define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 10) +#define DEFAULT_BUCKET_SIZE 131072 +#define DEFAULT_MAP_CAPACITY 131072 +#define DEFAULT_MAP_SIZE (DEFAULT_MAP_CAPACITY * 100) #define ROWS_PER_MILLISECOND 1 #define MAX_NUM_SCALABLE_BF 100000 #define MIN_NUM_SCALABLE_BF 10 @@ -44,8 +44,8 @@ static void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count) { } } -static void clearItemHelper(void* p) { - SScalableBf** pBf = p; +static void clearItemHelper(void *p) { + SScalableBf **pBf = p; tScalableBfDestroy(*pBf); } @@ -274,7 +274,7 @@ void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { } int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo) { - if(!pInfo) { + if (!pInfo) { return 0; } From e48e485b6c71165a1546cbdf4ecef37b9e599225 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 11:56:24 +0000 Subject: [PATCH 228/238] update default buffer --- source/libs/stream/src/streamBackendRocksdb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index e07198f6ab..5be6d1b003 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -572,11 +572,10 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { *dest = NULL; return -1; } - int64_t now = taosGetTimestampMs(); p = taosDecodeFixedI64(p, &key.unixTimestamp); p = taosDecodeFixedI32(p, &key.len); if (vlen != sizeof(int64_t) + sizeof(int32_t) + key.len) { - *dest = NULL; + if (dest != NULL) *dest = NULL; return -1; } @@ -587,6 +586,7 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { } if (ttl != NULL) { + int64_t now = taosGetTimestampMs(); *ttl = key.unixTimestamp == 0 ? 0 : key.unixTimestamp - now; } if (dest != NULL) { From 3ec700e0f7c20d036a87c50d140c08dd17b2edda Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 22:25:38 +0800 Subject: [PATCH 229/238] fix mem problem --- source/libs/stream/src/streamBackendRocksdb.c | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 5be6d1b003..6309cb9df2 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -576,6 +576,7 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { p = taosDecodeFixedI32(p, &key.len); if (vlen != sizeof(int64_t) + sizeof(int32_t) + key.len) { if (dest != NULL) *dest = NULL; + qError("vlen: %d, read len: %d", vlen, key.len); return -1; } @@ -1008,35 +1009,35 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[idx]); } -#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamGetInit(pState, funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = \ - ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ - char* ttlV = NULL; \ - int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ - rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ - if (err != NULL) { \ - taosMemoryFree(err); \ - qError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ - code = -1; \ - } else { \ - qTrace("streamState str:%s succ to write to %s, valLen:%d", toString, funcname, vLen); \ - } \ - taosMemoryFree(ttlV); \ +#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamGetInit(pState, funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = \ + ((rocksdb_column_family_handle_t**)pState->pTdbState->pHandle)[ginitDict[i].idx]; \ + rocksdb_t* db = pState->pTdbState->rocksdb; \ + rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \ + char* ttlV = NULL; \ + int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \ + rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \ + if (err != NULL) { \ + taosMemoryFree(err); \ + qError("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \ + code = -1; \ + } else { \ + qTrace("streamState str:%s succ to write to %s, rowValLen:%d, ttlValLen:%d", toString, funcname, vLen, ttlVLen); \ + } \ + taosMemoryFree(ttlV); \ } while (0); #define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ @@ -1071,17 +1072,17 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa code = -1; \ } else { \ char* p = NULL; \ - int32_t len = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ - if (len <= 0) { \ + int32_t tlen = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (tlen <= 0) { \ qError("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ funcname); \ code = -1; \ } else { \ qTrace("streamState str: %s succ to read from %s_%s, valLen:%d", toString, pState->pTdbState->idstr, funcname, \ - len); \ + tlen); \ } \ taosMemoryFree(val); \ - if (vLen != NULL) *vLen = len; \ + if (vLen != NULL) *vLen = tlen; \ } \ if (code == 0) \ qDebug("streamState str: %s succ to read from %s_%s", toString, pState->pTdbState->idstr, funcname); \ From b5b4cd2a05b686ab0de277d3354fad3f62c04f86 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 23:15:27 +0800 Subject: [PATCH 230/238] fix(query): load del info with upper version limitation. --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbDataIter.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 17 +++++++++++++---- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 12 ++++++++---- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++-- source/libs/wal/src/walRead.c | 2 +- 8 files changed, 28 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index faf550ab75..828e0e7e3d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -297,7 +297,7 @@ int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 6a97ea89b3..d6c19ee18e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1454,7 +1454,7 @@ static int32_t getTableDelDataFromDelIdx(SDelFReader *pDelReader, SDelIdx *pDelI int32_t code = 0; if (pDelIdx) { - code = tsdbReadDelData(pDelReader, pDelIdx, aDelData); + code = tsdbReadDelData(pDelReader, pDelIdx, aDelData, INT64_MAX); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d15f848cfd..ba74613bd9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -266,7 +266,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel suid = pDelIdx->suid; uid = pDelIdx->uid; - code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData); + code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); } else { taosArrayClear(pCommitter->aDelData); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataIter.c b/source/dnode/vnode/src/tsdb/tsdbDataIter.c index e27aec5b1b..701ad56181 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataIter.c @@ -412,7 +412,7 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* } } - code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData); + code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); pIter->delInfo.suid = pDelIdx->suid; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9cae4ad5aa..1286c2f7c4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2967,7 +2967,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SDelIdx* pIdx = taosArraySearch(pReader->pDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData); + code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData, pReader->verRange.maxVer); } if (code != TSDB_CODE_SUCCESS) { goto _err; @@ -2978,7 +2978,10 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* if (pMemTbData != NULL) { p = pMemTbData->pHead; while (p) { - taosArrayPush(pDelData, p); + if (p->version <= pReader->verRange.maxVer) { + taosArrayPush(pDelData, p); + } + p = p->pNext; } } @@ -2986,7 +2989,9 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* if (piMemTbData != NULL) { p = piMemTbData->pHead; while (p) { - taosArrayPush(pDelData, p); + if (p->version <= pReader->verRange.maxVer) { + taosArrayPush(pDelData, p); + } p = p->pNext; } } @@ -4558,7 +4563,11 @@ int32_t tsdbReaderOpen(void* pVnode, SQueryTableDataCond* pCond, void* pTableLis pReader->pIgnoreTables = pIgnoreTables; - tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); + tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64 + " in this query %s", + pReader, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->verRange.minVer, + pReader->verRange.maxVer, pReader->idStr); + return code; _err: diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index b25ab393da..e4b8a57291 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1488,7 +1488,7 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { return code; } -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t code = 0; int64_t offset = pDelIdx->offset; int64_t size = pDelIdx->size; @@ -1510,11 +1510,15 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData SDelData delData; n += tGetDelData(pReader->aBuf[0] + n, &delData); - if (taosArrayPush(aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + if (delData.version > maxVer) { + continue; } + if (taosArrayPush(aDelData, &delData) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } + ASSERT(n == size); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index dfea125cc1..f7362b366f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1166,7 +1166,7 @@ static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* int32_t c = tTABLEIDCmprFn(pDelIdx, &pWriter->tbid); if (c < 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData); + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); SDelIdx* pDelIdxNew = taosArrayReserve(pWriter->aDelIdx, 1); @@ -1183,7 +1183,7 @@ static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pWriter->pTIter->tIter.iDelIdx++; } else if (c == 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); pWriter->pTIter->tIter.iDelIdx++; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 0eb3d6ef09..1223e3756c 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -333,7 +333,7 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { return -1; } - wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); + wDebug("vgId:%d, index:%" PRId64 " is fetched, type:%d, cursor advance", pReader->pWal->cfg.vgId, ver, pReader->pHead->head.msgType); pReader->curVersion = ver + 1; return 0; } From becacbc84b63a181758d16a0b033a89003d240b2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 23:36:47 +0800 Subject: [PATCH 231/238] fix(query): fix syntax error. --- source/dnode/vnode/src/inc/tsdb.h | 3 ++- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- source/dnode/vnode/src/tsdb/tsdbDataIter.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 4 ++-- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 828e0e7e3d..9df95a379a 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -297,7 +297,8 @@ int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter); // SDelFReader int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFReaderClose(SDelFReader **ppReader); -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer); +int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer); +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d6c19ee18e..c659c8f4a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1454,7 +1454,7 @@ static int32_t getTableDelDataFromDelIdx(SDelFReader *pDelReader, SDelIdx *pDelI int32_t code = 0; if (pDelIdx) { - code = tsdbReadDelData(pDelReader, pDelIdx, aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pDelReader, pDelIdx, aDelData, INT64_MAX); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index ba74613bd9..b440d51883 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -266,7 +266,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel suid = pDelIdx->suid; uid = pDelIdx->uid; - code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); } else { taosArrayClear(pCommitter->aDelData); diff --git a/source/dnode/vnode/src/tsdb/tsdbDataIter.c b/source/dnode/vnode/src/tsdb/tsdbDataIter.c index 701ad56181..8215c1ac29 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataIter.c @@ -412,7 +412,7 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* } } - code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); pIter->delInfo.suid = pDelIdx->suid; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1286c2f7c4..2500015ec1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2967,7 +2967,7 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* SDelIdx* pIdx = taosArraySearch(pReader->pDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData, pReader->verRange.maxVer); + code = tsdbReadDelDatav1(pReader->pDelFReader, pIdx, pDelData, pReader->verRange.maxVer); } if (code != TSDB_CODE_SUCCESS) { goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index e4b8a57291..fb899fa1ce 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1488,7 +1488,7 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { return code; } -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { +int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t code = 0; int64_t offset = pDelIdx->offset; int64_t size = pDelIdx->size; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index f7362b366f..b5ca716701 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -1166,7 +1166,7 @@ static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* int32_t c = tTABLEIDCmprFn(pDelIdx, &pWriter->tbid); if (c < 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pWriter->pDelFReader, pDelIdx, pWriter->pTIter->tIter.aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); SDelIdx* pDelIdxNew = taosArrayReserve(pWriter->aDelIdx, 1); @@ -1183,7 +1183,7 @@ static int32_t tsdbSnapWriteDelTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pWriter->pTIter->tIter.iDelIdx++; } else if (c == 0) { - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, INT64_MAX); + code = tsdbReadDelDatav1(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, INT64_MAX); TSDB_CHECK_CODE(code, lino, _exit); pWriter->pTIter->tIter.iDelIdx++; From 5063c175f4ff62ceb210d5424d1d7bda5681d1b9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 2 Jun 2023 23:47:14 +0800 Subject: [PATCH 232/238] fix mem problem --- source/libs/stream/src/streamBackendRocksdb.c | 2 +- source/libs/stream/src/tstreamFileState.c | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 6309cb9df2..b3995f020b 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -574,7 +574,7 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) { } p = taosDecodeFixedI64(p, &key.unixTimestamp); p = taosDecodeFixedI32(p, &key.len); - if (vlen != sizeof(int64_t) + sizeof(int32_t) + key.len) { + if (vlen != (sizeof(int64_t) + sizeof(int32_t) + key.len)) { if (dest != NULL) *dest = NULL; qError("vlen: %d, read len: %d", vlen, key.len); return -1; diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 7ea1a41925..757b1de6b6 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -419,7 +419,9 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (code != 0 || len == 0 || val == NULL) { return TSDB_CODE_FAILED; } - sscanf(val, "%" PRId64 "", &maxCheckPointId); + memcpy(val, buf, len); + buf[len] = 0; + maxCheckPointId = atol((char*)buf); taosMemoryFree(val); } for (int64_t i = maxCheckPointId; i > 0; i--) { @@ -431,16 +433,17 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { if (code != 0) { return TSDB_CODE_FAILED; } + memcpy(val, buf, len); + buf[len] = 0; + taosMemoryFree(val); + TSKEY ts; - sscanf(val, "%" PRId64 "", &ts); + ts = atol((char*)buf); if (ts < mark) { // statekey winkey.ts < mark forceRemoveCheckpoint(pFileState, i); - taosMemoryFreeClear(val); break; - } else { } - taosMemoryFree(val); } return code; } From 4bb64292498390fd17abb5fc5e141a1c2b34920c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Jun 2023 00:20:59 +0800 Subject: [PATCH 233/238] fix(query): fix syntax error. --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index fb899fa1ce..4b677533e7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1488,6 +1488,10 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { return code; } +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { + return tsdbReadDelDatav1(pReader, pDelIdx, aDelData, INT64_MAX); +} + int32_t tsdbReadDelDatav1(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData, int64_t maxVer) { int32_t code = 0; int64_t offset = pDelIdx->offset; From 81fe174aa26b0dd75ed9f805698edf919ed34ef3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 3 Jun 2023 08:19:59 +0800 Subject: [PATCH 234/238] fix mem leak --- source/libs/stream/src/streamExec.c | 39 ++++++++++++++++------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 716b939e5f..1a78607770 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -16,11 +16,11 @@ #include "streamInc.h" // maximum allowed processed block batches. One block may include several submit blocks -#define MAX_STREAM_EXEC_BATCH_NUM 32 -#define MIN_STREAM_EXEC_BATCH_NUM 8 -#define MAX_STREAM_RESULT_DUMP_THRESHOLD 100 +#define MAX_STREAM_EXEC_BATCH_NUM 32 +#define MIN_STREAM_EXEC_BATCH_NUM 8 +#define MAX_STREAM_RESULT_DUMP_THRESHOLD 100 -static int32_t updateCheckPointInfo (SStreamTask* pTask); +static int32_t updateCheckPointInfo(SStreamTask* pTask); bool streamTaskShouldStop(const SStreamStatus* pStatus) { int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus); @@ -48,10 +48,11 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return -1; } - qDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, size/1048576.0); + qDebug("s-task:%s dump stream result data blocks, num:%d, size:%.2fMiB", pTask->id.idStr, numOfBlocks, + size / 1048576.0); code = streamTaskOutputResultBlock(pTask, pStreamBlocks); - if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { // back pressure and record position + if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { // back pressure and record position destroyStreamDataBlock(pStreamBlocks); return -1; } @@ -65,7 +66,8 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return TSDB_CODE_SUCCESS; } -static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { +static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, + int32_t* totalBlocks) { int32_t code = TSDB_CODE_SUCCESS; void* pExecutor = pTask->exec.pExecutor; @@ -82,7 +84,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i } if (streamTaskShouldStop(&pTask->status)) { - taosArrayDestroy(pRes); // memory leak + taosArrayDestroy(pRes); // memory leak return 0; } @@ -101,7 +103,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SSDataBlock block = {0}; - const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*) pItem; + const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); @@ -153,7 +155,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i ASSERT(numOfBlocks == taosArrayGetSize(pRes)); code = doDumpResult(pTask, pItem, pRes, size, totalSize, totalBlocks); } else { - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); } return code; @@ -286,7 +288,7 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { } #endif -int32_t updateCheckPointInfo (SStreamTask* pTask) { +int32_t updateCheckPointInfo(SStreamTask* pTask) { int64_t ckId = 0; int64_t dataVer = 0; qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId); @@ -294,7 +296,8 @@ int32_t updateCheckPointInfo (SStreamTask* pTask) { SCheckpointInfo* pCkInfo = &pTask->chkInfo; if (ckId > pCkInfo->id) { // save it since the checkpoint is updated qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64 - ", checkPoint id:%" PRId64 " -> %" PRId64, pTask->id.idStr, pCkInfo->version, dataVer, pCkInfo->id, ckId); + ", checkPoint id:%" PRId64 " -> %" PRId64, + pTask->id.idStr, pCkInfo->version, dataVer, pCkInfo->id, ckId); pTask->chkInfo = (SCheckpointInfo){.version = dataVer, .id = ckId, .currentVer = pCkInfo->currentVer}; @@ -417,14 +420,15 @@ int32_t streamExecForAll(SStreamTask* pTask) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)pInput; qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); - qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, - pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); + qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, + pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)pInput; SArray* pBlockList = pBlock->blocks; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); + qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, + pBlock->sourceVer); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)pInput; @@ -445,8 +449,9 @@ int32_t streamExecForAll(SStreamTask* pTask) { int32_t totalBlocks = 0; streamTaskExecImpl(pTask, pInput, &resSize, &totalBlocks); - double el = (taosGetTimestampMs() - st) / 1000.0; - qDebug("s-task:%s exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", pTask->id.idStr, el, resSize / 1048576.0, totalBlocks); + double el = (taosGetTimestampMs() - st) / 1000.0; + qDebug("s-task:%s exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", pTask->id.idStr, el, + resSize / 1048576.0, totalBlocks); streamFreeQitem(pInput); } From 7244ab1f6bfd28c1d5d2a9cbc4f4442a8bb50c94 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 3 Jun 2023 08:28:21 +0800 Subject: [PATCH 235/238] fix mem leak --- source/libs/stream/src/streamExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 1a78607770..aa2c8050de 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -84,7 +84,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i } if (streamTaskShouldStop(&pTask->status)) { - taosArrayDestroy(pRes); // memory leak + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); return 0; } From 9e0392ff59a1db9bcee81c01dc7b30cc2fdda8ff Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 3 Jun 2023 09:17:05 +0800 Subject: [PATCH 236/238] fix mem leak --- source/libs/stream/src/streamExec.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index aa2c8050de..49b35ce8db 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -101,8 +101,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; - + SSDataBlock block = {0}; const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)pItem; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); From 0f69aa0b7a65edb0c2f19b42487693d0e812dbd3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 3 Jun 2023 17:24:17 +0800 Subject: [PATCH 237/238] other: set the tags for 3.0 branch. --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9bbda8309f..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From d02746d86f3773664e990490e767ed2b717ed8e1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 5 Jun 2023 15:24:39 +0800 Subject: [PATCH 238/238] fix(query/sys): pause/resume with sys scan operator --- include/libs/executor/storageapi.h | 218 +++++++++++---------- source/dnode/vnode/src/inc/vnodeInt.h | 17 +- source/dnode/vnode/src/meta/metaQuery.c | 78 +++++--- source/dnode/vnode/src/vnd/vnodeInitApi.c | 28 +-- source/libs/executor/src/sysscanoperator.c | 43 ++-- 5 files changed, 221 insertions(+), 163 deletions(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 4cfbf2c2fa..e2944d13da 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -16,13 +16,13 @@ #ifndef TDENGINE_STORAGEAPI_H #define TDENGINE_STORAGEAPI_H -#include "tsimplehash.h" -#include "tscalablebf.h" -#include "taosdef.h" -#include "tmsg.h" -#include "tcommon.h" -#include "index.h" #include "function.h" +#include "index.h" +#include "taosdef.h" +#include "tcommon.h" +#include "tmsg.h" +#include "tscalablebf.h" +#include "tsimplehash.h" #ifdef __cplusplus extern "C" { @@ -46,7 +46,7 @@ typedef struct SMetaEntry { int8_t type; int8_t flags; // TODO: need refactor? tb_uid_t uid; - char * name; + char* name; union { struct { SSchemaWrapper schemaRow; @@ -57,43 +57,45 @@ typedef struct SMetaEntry { int64_t ctime; int32_t ttlDays; int32_t commentLen; - char * comment; + char* comment; tb_uid_t suid; - uint8_t *pTags; + uint8_t* pTags; } ctbEntry; struct { int64_t ctime; int32_t ttlDays; int32_t commentLen; - char * comment; + char* comment; int32_t ncid; // next column id SSchemaWrapper schemaRow; } ntbEntry; struct { - STSma *tsma; + STSma* tsma; } smaEntry; }; - uint8_t *pBuf; + uint8_t* pBuf; } SMetaEntry; typedef struct SMetaReader { - int32_t flags; - void * pMeta; - SDecoder coder; - SMetaEntry me; - void * pBuf; - int32_t szBuf; - struct SStoreMeta* pAPI; + int32_t flags; + void* pMeta; + SDecoder coder; + SMetaEntry me; + void* pBuf; + int32_t szBuf; + struct SStoreMeta* pAPI; } SMetaReader; typedef struct SMTbCursor { - void * pDbc; - void * pKey; - void * pVal; + void* pMeta; + void* pDbc; + void* pKey; + void* pVal; int32_t kLen; int32_t vLen; SMetaReader mr; + int8_t paused; } SMTbCursor; typedef struct SRowBuffPos { @@ -107,22 +109,22 @@ typedef struct SRowBuffPos { typedef struct SMetaTableInfo { int64_t suid; int64_t uid; - SSchemaWrapper *schema; + SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; } SMetaTableInfo; typedef struct SSnapContext { - SMeta * pMeta; // todo remove it - int64_t snapVersion; - void * pCur; - int64_t suid; - int8_t subType; - SHashObj * idVersion; - SHashObj * suidInfo; - SArray * idList; - int32_t index; - bool withMeta; - bool queryMeta; // true-get meta, false-get data + SMeta* pMeta; // todo remove it + int64_t snapVersion; + void* pCur; + int64_t suid; + int8_t subType; + SHashObj* idVersion; + SHashObj* suidInfo; + SArray* idList; + int32_t index; + bool withMeta; + bool queryMeta; // true-get meta, false-get data } SSnapContext; typedef struct { @@ -139,10 +141,9 @@ typedef struct { // int32_t tqReaderSeek(STqReader *pReader, int64_t ver, const char *id); // bool tqNextBlockInWal(STqReader* pReader, const char* idstr); // bool tqNextBlockImpl(STqReader *pReader, const char* idstr); -// int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid); -// SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); -// int32_t setForSnapShot(SSnapContext *ctx, int64_t uid); -// int32_t destroySnapContext(SSnapContext *ctx); +// int32_t getTableInfoFromSnapshot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t +// *uid); SMetaTableInfo getMetaTableInfoFromSnapshot(SSnapContext *ctx); int32_t setForSnapShot(SSnapContext +// *ctx, int64_t uid); int32_t destroySnapContext(SSnapContext *ctx); // clang-format off /*-------------------------------------------------new api format---------------------------------------------------*/ @@ -219,16 +220,16 @@ typedef struct SStoreTqReader { bool (*tqReaderIsQueriedTable)(); bool (*tqReaderCurrentBlockConsumed)(); - struct SWalReader *(*tqReaderGetWalReader)(); // todo remove it - int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it + struct SWalReader* (*tqReaderGetWalReader)(); // todo remove it + int32_t (*tqReaderRetrieveTaosXBlock)(); // todo remove it int32_t (*tqReaderSetSubmitMsg)(); // todo remove it bool (*tqReaderNextBlockFilterOut)(); } SStoreTqReader; typedef struct SStoreSnapshotFn { - int32_t (*createSnapshot)(SSnapContext *ctx, int64_t uid); - int32_t (*destroySnapshot)(SSnapContext *ctx); + int32_t (*createSnapshot)(SSnapContext* ctx, int64_t uid); + int32_t (*destroySnapshot)(SSnapContext* ctx); SMetaTableInfo (*getMetaTableInfoFromSnapshot)(SSnapContext* ctx); int32_t (*getTableInfoFromSnapshot)(SSnapContext* ctx, void** pBuf, int32_t* contLen, int16_t* type, int64_t* uid); } SStoreSnapshotFn; @@ -252,42 +253,54 @@ int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, in int32_t payloadLen, double selectivityRatio); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); -int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); +int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t +payloadLen); */ typedef struct SStoreMeta { - SMTbCursor *(*openTableMetaCursor)(void *pVnode); // metaOpenTbCursor - void (*closeTableMetaCursor)(SMTbCursor *pTbCur); // metaCloseTbCursor - int32_t (*cursorNext)(SMTbCursor *pTbCur, ETableType jumpTableType); // metaTbCursorNext - int32_t (*cursorPrev)(SMTbCursor *pTbCur, ETableType jumpTableType); // metaTbCursorPrev + SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor + void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor + void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor + void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first); // metaResumeTbCursor + int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext + int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev - int32_t (*getTableTags)(void *pVnode, uint64_t suid, SArray *uidList); - int32_t (*getTableTagsByUid)(void *pVnode, int64_t suid, SArray *uidList); - const void *(*extractTagVal)(const void *tag, int16_t type, STagVal *tagVal); // todo remove it + int32_t (*getTableTags)(void* pVnode, uint64_t suid, SArray* uidList); + int32_t (*getTableTagsByUid)(void* pVnode, int64_t suid, SArray* uidList); + const void* (*extractTagVal)(const void* tag, int16_t type, STagVal* tagVal); // todo remove it - int32_t (*getTableUidByName)(void *pVnode, char *tbName, uint64_t *uid); - int32_t (*getTableTypeByName)(void *pVnode, char *tbName, ETableType *tbType); - int32_t (*getTableNameByUid)(void *pVnode, uint64_t uid, char *tbName); - bool (*isTableExisted)(void *pVnode, tb_uid_t uid); + int32_t (*getTableUidByName)(void* pVnode, char* tbName, uint64_t* uid); + int32_t (*getTableTypeByName)(void* pVnode, char* tbName, ETableType* tbType); + int32_t (*getTableNameByUid)(void* pVnode, uint64_t uid, char* tbName); + bool (*isTableExisted)(void* pVnode, tb_uid_t uid); - int32_t (*metaGetCachedTbGroup)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); - int32_t (*metaPutTbGroupToCache)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen); + int32_t (*metaGetCachedTbGroup)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); + int32_t (*metaPutTbGroupToCache)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen); - int32_t (*getCachedTableList)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes); - int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio); + int32_t (*getCachedTableList)(void* pVnode, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, + bool* acquireRes); + int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen, double selectivityRatio); - void *(*storeGetIndexInfo)(); - void *(*getInvertIndex)(void* pVnode); - int32_t (*getChildTableList)(void *pVnode, int64_t suid, SArray *list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] - int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList - void *storeGetVersionRange; - void *storeGetLastTimestamp; + void* (*storeGetIndexInfo)(); + void* (*getInvertIndex)(void* pVnode); + int32_t (*getChildTableList)( + void* pVnode, int64_t suid, + SArray* list); // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter] + int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList); // vnodeGetStbIdList & vnodeGetAllTableList + void* storeGetVersionRange; + void* storeGetLastTimestamp; - int32_t (*getTableSchema)(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); // tsdbGetTableSchema + int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema // db name, vgId, numOfTables, numOfSTables - int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); - void (*getBasicInfo)(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables);// vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); + int32_t (*getNumOfChildTables)( + void* pVnode, int64_t uid, + int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); + void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, + int64_t* numOfNormalTables); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & + // metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); int64_t (*getNumOfRowsInMem)(void* pVnode); /** @@ -298,24 +311,24 @@ int32_t vnodeGetStbIdList(void *pVnode, int64_t suid, SArray *list); } SStoreMeta; typedef struct SStoreMetaReader { - void (*initReader)(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta* pAPI); - void (*clearReader)(SMetaReader *pReader); - void (*readerReleaseLock)(SMetaReader *pReader); - int32_t (*getTableEntryByUid)(SMetaReader *pReader, tb_uid_t uid); - int32_t (*getTableEntryByName)(SMetaReader *pReader, const char *name); - int32_t (*getEntryGetUidCache)(SMetaReader *pReader, tb_uid_t uid); + void (*initReader)(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI); + void (*clearReader)(SMetaReader* pReader); + void (*readerReleaseLock)(SMetaReader* pReader); + int32_t (*getTableEntryByUid)(SMetaReader* pReader, tb_uid_t uid); + int32_t (*getTableEntryByName)(SMetaReader* pReader, const char* name); + int32_t (*getEntryGetUidCache)(SMetaReader* pReader, tb_uid_t uid); } SStoreMetaReader; typedef struct SUpdateInfo { - SArray *pTsBuckets; + SArray* pTsBuckets; uint64_t numBuckets; - SArray *pTsSBFs; + SArray* pTsSBFs; uint64_t numSBFs; int64_t interval; int64_t watermark; TSKEY minTS; - SScalableBf *pCloseWinSBF; - SHashObj *pMap; + SScalableBf* pCloseWinSBF; + SHashObj* pMap; uint64_t maxDataVersion; } SUpdateInfo; @@ -334,15 +347,15 @@ typedef struct SStateStore { int32_t (*streamStateAddIfNotExist)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateReleaseBuf)(SStreamState* pState, const SWinKey* key, void* pVal); - void (*streamStateFreeVal)(void* val); + void (*streamStateFreeVal)(void* val); int32_t (*streamStatePut)(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t (*streamStateGet)(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); - bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); + bool (*streamStateCheck)(SStreamState* pState, const SWinKey* key); int32_t (*streamStateGetByPos)(SStreamState* pState, void* pos, void** pVal); int32_t (*streamStateDel)(SStreamState* pState, const SWinKey* key); int32_t (*streamStateClear)(SStreamState* pState); - void (*streamStateSetNumber)(SStreamState* pState, int32_t number); + void (*streamStateSetNumber)(SStreamState* pState, int32_t number); int32_t (*streamStateSaveInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen); int32_t (*streamStateGetInfo)(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen); @@ -353,36 +366,37 @@ typedef struct SStateStore { int32_t (*streamStateCurNext)(SStreamState* pState, SStreamStateCur* pCur); int32_t (*streamStateCurPrev)(SStreamState* pState, SStreamStateCur* pCur); - SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); - SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); - SStreamStateCur* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); - SStreamStateCur* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); - void (*streamStateFreeCur)(SStreamStateCur* pCur); + SStreamStateCur* (*streamStateGetAndCheckCur)(SStreamState* pState, SWinKey* key); + SStreamStateCur* (*streamStateSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyNext)(SStreamState* pState, const SWinKey* key); + SStreamStateCur* (*streamStateFillSeekKeyPrev)(SStreamState* pState, const SWinKey* key); + void (*streamStateFreeCur)(SStreamStateCur* pCur); int32_t (*streamStateGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); - int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen); + int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, + int32_t* pVLen); int32_t (*streamStateSessionPut)(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); int32_t (*streamStateSessionGet)(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionDel)(SStreamState* pState, const SSessionKey* key); int32_t (*streamStateSessionClear)(SStreamState* pState); int32_t (*streamStateSessionGetKVByCur)(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, - state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); - SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); - TSKEY (*updateInfoFillBlockData)(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol); - bool (*updateInfoIsUpdated)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts); - bool (*updateInfoIsTableInserted)(SUpdateInfo *pInfo, int64_t tbUid); - void (*updateInfoDestroy)(SUpdateInfo *pInfo); + SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark); + TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol); + bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts); + bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); + void (*updateInfoDestroy)(SUpdateInfo* pInfo); - SUpdateInfo* (*updateInfoInitP)(SInterval *pInterval, int64_t watermark); - void (*updateInfoAddCloseWindowSBF)(SUpdateInfo *pInfo); - void (*updateInfoDestoryColseWinSBF)(SUpdateInfo *pInfo); - int32_t (*updateInfoSerialize)(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); - int32_t (*updateInfoDeserialize)(void *buf, int32_t bufLen, SUpdateInfo *pInfo); + SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark); + void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); + void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); + int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); + int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo); SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); SStreamStateCur* (*streamStateSessionSeekKeyCurrentPrev)(SStreamState* pState, const SSessionKey* key); @@ -396,11 +410,11 @@ typedef struct SStateStore { bool (*needClearDiskBuff)(struct SStreamFileState* pFileState); SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); - void (*streamStateClose)(SStreamState* pState, bool remove); - int32_t (*streamStateBegin)(SStreamState* pState); - int32_t (*streamStateCommit)(SStreamState* pState); - void (*streamStateDestroy)(SStreamState* pState, bool remove); - int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); + void (*streamStateClose)(SStreamState* pState, bool remove); + int32_t (*streamStateBegin)(SStreamState* pState); + int32_t (*streamStateCommit)(SStreamState* pState); + void (*streamStateDestroy)(SStreamState* pState, bool remove); + int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark); } SStateStore; typedef struct SStorageAPI { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index db285dc124..96486ec317 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -107,10 +107,12 @@ struct SQueryNode { typedef SVCreateTbReq STbCfg; typedef SVCreateTSmaReq SSmaCfg; -SMTbCursor *metaOpenTbCursor(void *pVnode); -void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); +SMTbCursor* metaOpenTbCursor(void* pVnode); +void metaCloseTbCursor(SMTbCursor* pTbCur); +void metaPauseTbCursor(SMTbCursor* pTbCur); +void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first); +int32_t metaTbCursorNext(SMTbCursor* pTbCur, ETableType jumpTableType); +int32_t metaTbCursorPrev(SMTbCursor* pTbCur, ETableType jumpTableType); #endif @@ -154,8 +156,8 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); -int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); -int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); +int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid); +int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); @@ -175,7 +177,7 @@ void* metaGetIdx(SMeta* pMeta); void* metaGetIvtIdx(SMeta* pMeta); int metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList); -void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); +void metaReaderInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -491,7 +493,6 @@ struct SCompactInfo { void initStorageAPI(SStorageAPI* pAPI); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index fa9eea5e29..29fe89c3f2 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -17,8 +17,8 @@ #include "osMemory.h" #include "tencode.h" -void _metaReaderInit(SMetaReader* pReader, void* pVnode, int32_t flags, SStoreMeta* pAPI) { - SMeta* pMeta = ((SVnode*)pVnode)->pMeta; +void _metaReaderInit(SMetaReader *pReader, void *pVnode, int32_t flags, SStoreMeta *pAPI) { + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; metaReaderInit(pReader, pMeta, flags); pReader->pAPI = pAPI; } @@ -71,7 +71,7 @@ _err: } bool metaIsTableExist(void *pVnode, tb_uid_t uid) { - SVnode* pVnodeObj = pVnode; + SVnode *pVnodeObj = pVnode; metaRLock(pVnodeObj->pMeta); // query uid.idx if (tdbTbGet(pVnodeObj->pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { @@ -143,7 +143,7 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { int metaGetTableNameByUid(void *pVnode, uint64_t uid, char *tbName) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0); code = metaReaderGetTableEntryByUid(&mr, uid); if (code < 0) { metaReaderClear(&mr); @@ -179,7 +179,7 @@ int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) { SMetaReader *pReader = &mr; // query name.idx - if (tdbTbGet(((SMeta*)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(((SMeta *)pReader->pMeta)->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; metaReaderClear(&mr); return -1; @@ -195,7 +195,7 @@ int metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid) { int metaGetTableTypeByName(void *pVnode, char *tbName, ETableType *tbType) { int code = 0; SMetaReader mr = {0}; - metaReaderInit(&mr, ((SVnode*)pVnode)->pMeta, 0); + metaReaderInit(&mr, ((SVnode *)pVnode)->pMeta, 0); code = metaGetTableEntryByName(&mr, tbName); if (code == 0) *tbType = mr.me.type; @@ -221,12 +221,13 @@ SMTbCursor *metaOpenTbCursor(void *pVnode) { return NULL; } - SVnode* pVnodeObj = pVnode; - metaReaderInit(&pTbCur->mr, pVnodeObj->pMeta, 0); + SVnode *pVnodeObj = pVnode; + // metaReaderInit(&pTbCur->mr, pVnodeObj->pMeta, 0); - tdbTbcOpen(pVnodeObj->pMeta->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); - - tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + // tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + pTbCur->pMeta = pVnodeObj->pMeta; + pTbCur->paused = 1; + metaResumeTbCursor(pTbCur, 1); return pTbCur; } @@ -234,14 +235,45 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { if (pTbCur) { tdbFree(pTbCur->pKey); tdbFree(pTbCur->pVal); - metaReaderClear(&pTbCur->mr); - if (pTbCur->pDbc) { - tdbTbcClose((TBC *)pTbCur->pDbc); + if (!pTbCur->paused) { + metaReaderClear(&pTbCur->mr); + if (pTbCur->pDbc) { + tdbTbcClose((TBC *)pTbCur->pDbc); + } } taosMemoryFree(pTbCur); } } +void metaPauseTbCursor(SMTbCursor *pTbCur) { + if (!pTbCur->paused) { + metaReaderClear(&pTbCur->mr); + tdbTbcClose((TBC *)pTbCur->pDbc); + pTbCur->paused = 1; + } +} +void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first) { + if (pTbCur->paused) { + metaReaderInit(&pTbCur->mr, pTbCur->pMeta, 0); + + tdbTbcOpen(((SMeta *)pTbCur->pMeta)->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); + + if (first) { + tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + } else { + int c = 0; + tdbTbcMoveTo(pTbCur->pDbc, pTbCur->pKey, pTbCur->kLen, &c); + if (c < 0) { + tdbTbcMoveToPrev(pTbCur->pDbc); + } else { + tdbTbcMoveToNext(pTbCur->pDbc); + } + } + + pTbCur->paused = 0; + } +} + int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { int ret; void *pBuf; @@ -974,7 +1006,7 @@ typedef struct { } SIdxCursor; int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) { - SMeta *pMeta = ((SVnode*)pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; @@ -1034,7 +1066,7 @@ END: } int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) { - SMeta *pMeta = ((SVnode*)pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; char *buf = NULL; @@ -1101,7 +1133,7 @@ END: return ret; } int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) { - SMeta *pMeta = ((SVnode*)pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMetaFltParam *param = arg; int32_t ret = 0; char *buf = NULL; @@ -1132,7 +1164,7 @@ END: return 0; } int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) { - SMeta *pMeta = ((SVnode*)pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMetaFltParam *param = arg; SMetaEntry oStbEntry = {0}; @@ -1318,7 +1350,7 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi } int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) { - SMeta* pMeta = ((SVnode*) pVnode)->pMeta; + SMeta *pMeta = ((SVnode *)pVnode)->pMeta; const int32_t LIMIT = 128; int32_t isLock = false; @@ -1350,8 +1382,8 @@ int32_t metaGetTableTagsByUids(void *pVnode, int64_t suid, SArray *uidList) { return 0; } -int32_t metaGetTableTags(void* pVnode, uint64_t suid, SArray *pUidTagInfo) { - SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode*)pVnode)->pMeta, suid, 1); +int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { + SMCtbCursor *pCur = metaOpenCtbCursor(((SVnode *)pVnode)->pMeta, suid, 1); // If len > 0 means there already have uids, and we only want the // tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept @@ -1456,11 +1488,11 @@ _exit: return code; } -int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t* numOfTables) { +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables) { int32_t code = 0; *numOfTables = 0; - SVnode* pVnodeObj = pVnode; + SVnode *pVnodeObj = pVnode; metaRLock(pVnodeObj->pMeta); // fast path: search cache diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index 526b9b4e2d..d2db6368a2 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#include "storageapi.h" -#include "vnodeInt.h" -#include "tstreamUpdate.h" #include "meta.h" +#include "storageapi.h" +#include "tstreamUpdate.h" +#include "vnodeInt.h" static void initTsdbReaderAPI(TsdReader* pReader); static void initMetadataAPI(SStoreMeta* pMeta); @@ -56,10 +56,10 @@ void initTsdbReaderAPI(TsdReader* pReader) { pReader->tsdReaderResetStatus = tsdbReaderReset; pReader->tsdReaderGetDataBlockDistInfo = tsdbGetFileBlocksDistInfo; - pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away + pReader->tsdReaderGetNumOfInMemRows = tsdbGetNumOfRowsInMemTable; // todo this function should be moved away pReader->tsdSetQueryTableList = tsdbSetTableList; - pReader->tsdSetReaderTaskId = (void (*)(void *, const char *))tsdbReaderSetId; + pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId; } void initMetadataAPI(SStoreMeta* pMeta) { @@ -67,6 +67,8 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->openTableMetaCursor = metaOpenTbCursor; pMeta->closeTableMetaCursor = metaCloseTbCursor; + pMeta->pauseTableMetaCursor = metaPauseTbCursor; + pMeta->resumeTableMetaCursor = metaResumeTbCursor; pMeta->cursorNext = metaTbCursorNext; pMeta->cursorPrev = metaTbCursorPrev; @@ -78,7 +80,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->storeGetIndexInfo = vnodeGetIdx; pMeta->getInvertIndex = vnodeGetIvtIdx; - pMeta->extractTagVal = (const void *(*)(const void *, int16_t, STagVal *))metaGetTableTagVal; + pMeta->extractTagVal = (const void* (*)(const void*, int16_t, STagVal*))metaGetTableTagVal; pMeta->getTableTags = metaGetTableTags; pMeta->getTableTagsByUid = metaGetTableTagsByUids; @@ -86,7 +88,7 @@ void initMetadataAPI(SStoreMeta* pMeta) { pMeta->getTableTypeByName = metaGetTableTypeByName; pMeta->getTableNameByUid = metaGetTableNameByUid; - pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor + pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor pMeta->storeGetTableList = vnodeGetTableList; pMeta->getCachedTableList = metaGetCachedTableUidList; @@ -106,7 +108,7 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderNextBlockInWal = tqNextBlockInWal; - pTq->tqNextBlockImpl = tqNextBlockImpl;// todo remove it + pTq->tqNextBlockImpl = tqNextBlockImpl; // todo remove it pTq->tqReaderAddTables = tqReaderAddTbUidList; pTq->tqReaderSetQueryTableList = tqReaderSetTbUidList; @@ -116,10 +118,10 @@ void initTqAPI(SStoreTqReader* pTq) { pTq->tqReaderIsQueriedTable = tqReaderIsQueriedTable; pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed; - pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it - pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it + pTq->tqReaderGetWalReader = tqGetWalReader; // todo remove it + pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock; // todo remove it - pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it + pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg; // todo remove it pTq->tqGetResultBlock = tqGetResultBlock; pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut; @@ -199,7 +201,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateClose = streamStateClose; pStore->streamStateBegin = streamStateBegin; pStore->streamStateCommit = streamStateCommit; - pStore->streamStateDestroy= streamStateDestroy; + pStore->streamStateDestroy = streamStateDestroy; pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint; } @@ -239,4 +241,4 @@ void initSnapshotFn(SStoreSnapshotFn* pSnapshot) { pSnapshot->destroySnapshot = destroySnapContext; pSnapshot->getMetaTableInfoFromSnapshot = getMetaTableInfoFromSnapshot; pSnapshot->getTableInfoFromSnapshot = getTableInfoFromSnapshot; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index fd766e21f8..23a7d2c9e9 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -25,6 +25,7 @@ #include "tdatablock.h" #include "tmsg.h" +#include "index.h" #include "operator.h" #include "query.h" #include "querytask.h" @@ -32,7 +33,6 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "index.h" typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); @@ -540,12 +540,12 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { int32_t restore = pInfo->restore; pInfo->restore = false; - + while (restore || ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { if (restore) { restore = false; } - + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -626,8 +626,8 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSysTableScanInfo* pInfo = pOperator->info; if (pOperator->status == OP_EXEC_DONE) { @@ -1100,8 +1100,8 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { } static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSysTableScanInfo* pInfo = pOperator->info; @@ -1288,11 +1288,16 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; + int8_t firstMetaCursor = 0; SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); + firstMetaCursor = 1; + } + if (!firstMetaCursor) { + pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0); } blockDataCleanup(pInfo->pRes); @@ -1436,12 +1441,14 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { numOfRows = 0; if (pInfo->pRes->info.rows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); break; } } } if (numOfRows > 0) { + pAPI->metaFn.pauseTableMetaCursor(pInfo->pCur); p->info.rows = numOfRows; pInfo->pRes->info.rows = numOfRows; @@ -1485,7 +1492,8 @@ static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { } else { if (pInfo->showRewrite == false) { if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI}; + SSTabFltArg arg = { + .pMeta = pInfo->readHandle.vnode, .pVnode = pInfo->readHandle.vnode, .pAPI = &pTaskInfo->storageAPI}; SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); idx->init = 0; @@ -1827,7 +1835,7 @@ void destroySysScanOperator(void* param) { pInfo->pIdx = NULL; } - if(pInfo->pSchema) { + if (pInfo->pSchema) { taosHashCleanup(pInfo->pSchema); pInfo->pSchema = NULL; } @@ -2144,7 +2152,7 @@ static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { return -1; } -static int32_t doGetTableRowSize(SReadHandle *pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) { +static int32_t doGetTableRowSize(SReadHandle* pHandle, uint64_t uid, int32_t* rowLen, const char* idstr) { *rowLen = 0; SMetaReader mr = {0}; @@ -2194,17 +2202,17 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { SBlockDistInfo* pBlockScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SStorageAPI* pAPI = &pTaskInfo->storageAPI; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - int32_t code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid, - (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); + int32_t code = doGetTableRowSize(&pBlockScanInfo->readHandle, pBlockScanInfo->uid, (int32_t*)&blockDistInfo.rowSize, + GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } pAPI->tsdReader.tsdReaderGetDataBlockDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); - blockDistInfo.numOfInmemRows = (int32_t) pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle); + blockDistInfo.numOfInmemRows = (int32_t)pAPI->tsdReader.tsdReaderGetNumOfInMemRows(pBlockScanInfo->pHandle); SSDataBlock* pBlock = pBlockScanInfo->pResBlock; @@ -2289,7 +2297,8 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); + code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, + (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; @@ -2316,4 +2325,4 @@ _error: taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); return NULL; -} \ No newline at end of file +}