From 0d9d2ea293c5b0679be29bdafa7f2ed5eab1188c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Feb 2023 18:31:32 +0800 Subject: [PATCH 001/106] fix(query): do some internal refactor, and fix the bug when calender time duration exists in sliding and offset. --- include/common/ttime.h | 2 +- include/libs/executor/executor.h | 4 +- source/common/src/ttime.c | 114 +++++++++++------- source/libs/executor/inc/executil.h | 3 + source/libs/executor/inc/executorimpl.h | 1 - source/libs/executor/src/executil.c | 37 +++++- source/libs/executor/src/executorimpl.c | 6 +- source/libs/executor/src/filloperator.c | 8 +- source/libs/executor/src/scanoperator.c | 37 +----- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timewindowoperator.c | 47 +------- source/libs/planner/src/planOptimizer.c | 4 +- 12 files changed, 129 insertions(+), 136 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index eaf44c2771..69a071d59f 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -74,7 +74,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 63e1c556de..382837951a 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -178,7 +178,9 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key); /** * return the scan info, in the form of tuple of two items, including table uid and current timestamp * @param tinfo diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 559ffd2aaf..bbb58cc63d 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -82,6 +82,7 @@ static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_ static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); +static int32_t parseTimezone(char* str, int64_t* tzOffset); static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = { parseLocaltime, parseLocaltimeDst}; @@ -708,21 +709,19 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati return getDuration(*duration, *unit, duration, timePrecision); } +#define IS_CALENDAR_TIME_DURATION(_d) (((_d) == 'n') || ((_d) == 'y')) + int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { return t; } - if (unit != 'n' && unit != 'y') { + if (!IS_CALENDAR_TIME_DURATION(unit)) { return t + duration; } // The following code handles the y/n time duration - int64_t numOfMonth = duration; - if (unit == 'y') { - numOfMonth *= 12; - } - + int64_t numOfMonth = (unit == 'y')? duration*12:duration; int64_t fraction = t % TSDB_TICK_PER_SECOND(precision); struct tm tm; @@ -764,13 +763,16 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char return (emon - smon) / (int32_t)interval; } -int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) { +int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { if (pInterval->sliding == 0 && pInterval->interval == 0) { - return t; + return ts; } - int64_t start = t; - if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') { + int64_t start = ts; + int32_t precision = pInterval->precision; + + if (IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { + start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; @@ -792,44 +794,72 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start = (int64_t)(taosMktime(&tm) * TSDB_TICK_PER_SECOND(precision)); } else { - int64_t delta = t - pInterval->interval; - int32_t factor = (delta >= 0) ? 1 : -1; + if (IS_CALENDAR_TIME_DURATION(pInterval->intervalUnit)) { + int64_t news = (ts / pInterval->sliding) * pInterval->sliding; + ASSERT(news <= ts); - start = (delta / pInterval->sliding + factor) * pInterval->sliding; + if (news <= ts) { + int64_t prev = news; + int64_t newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; - if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { - /* - * here we revised the start time of day according to the local time zone, - * but in case of DST, the start time of one day need to be dynamically decided. - */ - // todo refactor to extract function that is available for Linux/Windows/Mac platform -#if defined(WINDOWS) && _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char** tzname = _tzname; -#endif + if (newe < ts) { // move towards the greater endpoint + while(newe < ts && news < ts) { + news += pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } - start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); - } - - int64_t end = 0; - - // not enough time range - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - while (end < t) { // move forward to the correct time window - start += pInterval->sliding; - - if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { - end = start + pInterval->interval - 1; + prev = news; } else { - end = INT64_MAX; - break; + while (newe >= ts) { + prev = news; + news -= pInterval->sliding; + newe = taosTimeAdd(news, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } } + + return prev; } } else { - end = INT64_MAX; + int64_t delta = ts - pInterval->interval; + int32_t factor = (delta >= 0) ? 1 : -1; + + start = (delta / pInterval->sliding + factor) * pInterval->sliding; + + if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { + /* + * here we revised the start time of day according to the local time zone, + * but in case of DST, the start time of one day need to be dynamically decided. + */ + // todo refactor to extract function that is available for Linux/Windows/Mac platform +#if defined(WINDOWS) && _MSC_VER >= 1900 + // see + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char** tzname = _tzname; +#endif + + start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); + } + + int64_t end = 0; + + // not enough time range + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; + while (end < ts) { // move forward to the correct time window + start += pInterval->sliding; + + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { + end = start + pInterval->interval - 1; + } else { + end = INT64_MAX; + break; + } + } + } else { + end = INT64_MAX; + } } } @@ -842,7 +872,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; int64_t newEnd = end; - while (newEnd >= t) { + while (newEnd >= ts) { end = newEnd; newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f99c7de93d..20f43deaa3 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -164,4 +164,7 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, void printDataBlock(SSDataBlock* pBlock, const char* flag); +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 999a7965fb..1850cd007f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -854,7 +854,6 @@ int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResul SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 040e67713d..53947e85e7 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1686,12 +1686,12 @@ int32_t convertFillType(int32_t mode) { return type; } -static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { +void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery) { if (ascQuery) { - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - *w = getAlignQueryTimeWindow(pInterval, pInterval->precision, ts); + *w = getAlignQueryTimeWindow(pInterval, ts); int64_t key = w->skey; while (key < ts) { // moving towards end @@ -1708,7 +1708,7 @@ static void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindo static STimeWindow doCalculateTimeWindow(int64_t ts, SInterval* pInterval) { STimeWindow w = {0}; - w.skey = taosTimeTruncate(ts, pInterval, pInterval->precision); + w.skey = taosTimeTruncate(ts, pInterval); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return w; } @@ -1742,6 +1742,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI if (pRow) { w = pRow->win; } + // in case of typical time window, we can calculate time window directly. if (w.skey > ts || w.ekey < ts) { w = doCalculateTimeWindow(ts, pInterval); @@ -1756,6 +1757,34 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI return w; } +void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (pInterval->sliding != 'n' && pInterval->sliding != 'y') { + tw->skey += pInterval->sliding * factor; + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return; + } + + // convert key to second + int64_t key = convertTimePrecision(tw->skey, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + int64_t duration = pInterval->sliding; + if (pInterval->slidingUnit == 'y') { + duration *= 12; + } + + struct tm tm; + time_t t = (time_t) key; + taosLocalTime(&t, &tm); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + duration * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); + + tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; +} + bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { return (pLimitInfo->limit.limit != -1 || pLimitInfo->limit.offset != -1 || pLimitInfo->slimit.limit != -1 || pLimitInfo->slimit.offset != -1); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d28c3cfe58..464282613d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -637,15 +637,15 @@ bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code) ? tr void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; } ///////////////////////////////////////////////////////////////////////////////////////////// -STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) { +STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) { STimeWindow win = {0}; - win.skey = taosTimeTruncate(key, pInterval, precision); + win.skey = taosTimeTruncate(key, pInterval); /* * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; if (win.ekey < win.skey) { win.ekey = INT64_MAX; } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 16983cb507..3403c202b7 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -255,10 +255,10 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); + STimeWindow w = {0}; int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - STimeWindow w = getAlignQueryTimeWindow(pInterval, pInterval->precision, startKey); - w = getFirstQualifiedTimeWindow(startKey, &w, pInterval, order); + getInitialStartTimeWindow(pInterval, startKey, &w, order); pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); @@ -400,13 +400,13 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_ASC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); return win.skey; } TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) { STimeWindow win = {.skey = ts, .ekey = ts}; - getNextIntervalWindow(pInterval, &win, TSDB_ORDER_DESC); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC); return win.skey; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b556733254..c83c161326 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -67,39 +67,6 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); - - tw->ekey -= 1; -} - static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) { STimeWindow w = {0}; @@ -109,7 +76,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } if (order == TSDB_ORDER_ASC) { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey); ASSERT(w.ekey >= pBlockInfo->window.skey); if (w.ekey < pBlockInfo->window.ekey) { @@ -128,7 +95,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn } } } else { - w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey); + w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey); ASSERT(w.skey <= pBlockInfo->window.ekey); if (w.skey > pBlockInfo->window.skey) { diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 778281f9b4..ed786003d5 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -519,7 +519,7 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) pFillInfo->end = endKey; if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->interval.precision); + pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval); } pFillInfo->index = 0; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c5dc927bd1..15f803125c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -263,43 +263,6 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary return num; } -static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { - tw->skey += pInterval->sliding * factor; - tw->ekey = tw->skey + pInterval->interval - 1; - return; - } - - int64_t key = tw->skey, interval = pInterval->interval; - // convert key to second - key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - - if (pInterval->intervalUnit == 'y') { - interval *= 12; - } - - struct tm tm; - time_t t = (time_t)key; - taosLocalTime(&t, &tm); - - int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - mon = (int)(mon + interval); - tm.tm_year = mon / 12; - tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); - - tw->ekey -= 1; -} - -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) { - getNextTimeWindow(pInterval, pInterval->precision, order, tw); -} - void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { SqlFunctionCtx* pCtx = pSup->pCtx; @@ -455,7 +418,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; - getNextTimeWindow(pInterval, precision, order, pNext); + getNextTimeWindow(pInterval, pNext, order); // next time window is not in current block if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || @@ -500,7 +463,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, if (ascQuery && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -509,7 +472,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->skey = taosTimeTruncate(next, pInterval); pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; @@ -1380,7 +1343,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa do { if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i])) { - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); continue; } uint64_t winGpId = pGpDatas[i]; @@ -1397,7 +1360,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); } } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); } while (win.ekey <= endTsCols[i]); } } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 90a7261074..27c1555666 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1247,8 +1247,8 @@ static bool smaIndexOptEqualInterval(SScanLogicNode* pScan, SWindowLogicNode* pW .sliding = pIndex->sliding, .slidingUnit = pIndex->slidingUnit, .precision = pScan->node.precision}; - return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval, pScan->node.precision)) && - (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval, pScan->node.precision)); + return (pScan->scanRange.skey == taosTimeTruncate(pScan->scanRange.skey, &interval)) && + (pScan->scanRange.ekey + 1 == taosTimeTruncate(pScan->scanRange.ekey + 1, &interval)); } return true; } From 9b90416bf32effdd2617124394cedeab9dd2b721 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Feb 2023 18:38:08 +0800 Subject: [PATCH 002/106] test: add unit test case. --- source/libs/executor/test/timewindowTest.cpp | 141 +++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 source/libs/executor/test/timewindowTest.cpp diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp new file mode 100644 index 0000000000..6635c820fa --- /dev/null +++ b/source/libs/executor/test/timewindowTest.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taos.h" +#include "thash.h" +#include "tsimplehash.h" +#include "executor.h" +#include "ttime.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +namespace { +SInterval createInterval(int64_t interval, int64_t sliding, int64_t offset, char intervalUnit, char slidingUnit, + char offsetUnit, int8_t precision) { + SInterval v = {0}; + v.interval = interval; + v.intervalUnit = intervalUnit; + v.sliding = sliding; + v.slidingUnit = slidingUnit; + v.offset = offset; + v.offsetUnit = offsetUnit; + v.precision = precision; + return v; +} + +void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) { + char buf[64] = {0}; + char bufs[64] = {0}; + char bufe[64] = {0}; + + taosFormatUtcTime(buf, tListLen(buf), ts, precision); + + taosFormatUtcTime(bufs, tListLen(bufs), pWindow->skey, precision); + taosFormatUtcTime(bufe, tListLen(bufe), pWindow->ekey, precision); + + printf("%s [%s - %s]\n", buf, bufs, bufe); +} +} // namespace + +TEST(testCase, timewindow_gen) { + // time window + osSetTimezone("UTC"); + + SInterval interval = + createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + + int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms) + + STimeWindow w = {0}; + getInitialStartTimeWindow(&interval, key, &w, true); + printTimeWindow(&w, interval.precision, key); + + getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, interval.precision, key); + + printf("---------------------------------------------------\n"); + SInterval monthInterval = + createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&monthInterval, key, &w, true); + printTimeWindow(&w, monthInterval.precision, key); + + getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, monthInterval.precision, key); + + printf("----------------------------------------------------------\n"); + SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&slidingInterval, key, &w, true); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printf("next\n"); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, slidingInterval.precision, key); + + printf("----------------------------------------------------------\n"); + SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + int64_t k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true); + printTimeWindow(&w, calendar_interval_1n.precision, k1); + + printf("next\n"); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, calendar_interval_1n.precision, key); + + + printf("----------------------------------------------------------\n"); + SInterval calendar_interval_1d_sliding_1n = createInterval(1*86400*100L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + +} + +#pragma GCC diagnostic pop \ No newline at end of file From 579b1b0a6c142c26b6f14261076d7ec54aee7bd1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 Feb 2023 15:12:19 +0800 Subject: [PATCH 003/106] fix(query): check correct unit. --- source/libs/executor/src/executil.c | 2 +- source/libs/executor/test/timewindowTest.cpp | 65 ++++++++++++-------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 53947e85e7..91f384ce6a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1759,7 +1759,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->sliding != 'n' && pInterval->sliding != 'y') { + if (pInterval->slidingUnit != 'n' && pInterval->slidingUnit != 'y') { tw->skey += pInterval->sliding * factor; tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return; diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index 6635c820fa..be5e217ed1 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -56,85 +56,102 @@ void printTimeWindow(STimeWindow* pWindow, int8_t precision, int64_t ts) { } // namespace TEST(testCase, timewindow_gen) { - // time window + // set correct time zone osSetTimezone("UTC"); - + int32_t precision = TSDB_TIME_PRECISION_MILLI; + SInterval interval = - createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', TSDB_TIME_PRECISION_MILLI); + createInterval(10 * 86400 * 1000, 10 * 86400 * 1000, 0, 'd', 'd', 'd', precision); int64_t key = 1659312000L * 1000; // 2022-8-1 00:00:00 // UTC+8 (ms) STimeWindow w = {0}; getInitialStartTimeWindow(&interval, key, &w, true); - printTimeWindow(&w, interval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&interval, &w, TSDB_ORDER_ASC); printf("next\n"); - printTimeWindow(&w, interval.precision, key); + printTimeWindow(&w, precision, key); printf("---------------------------------------------------\n"); SInterval monthInterval = createInterval(1, 1, 0, 'n', 'n', 'd', TSDB_TIME_PRECISION_MILLI); getInitialStartTimeWindow(&monthInterval, key, &w, true); - printTimeWindow(&w, monthInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&monthInterval, &w, TSDB_ORDER_ASC); printf("next\n"); - printTimeWindow(&w, monthInterval.precision, key); + printTimeWindow(&w, precision, key); printf("----------------------------------------------------------\n"); SInterval slidingInterval = createInterval(1, 10*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); getInitialStartTimeWindow(&slidingInterval, key, &w, true); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); printf("next\n"); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&slidingInterval, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, slidingInterval.precision, key); + printTimeWindow(&w, precision, key); - printf("----------------------------------------------------------\n"); + printf("-----------------calendar_interval_1n_sliding_1d-------\n"); SInterval calendar_interval_1n = createInterval(1, 1*86400*1000, 0, 'n', 'd', 'd', TSDB_TIME_PRECISION_MILLI); int64_t k1 = 1664409600 * 1000L; getInitialStartTimeWindow(&calendar_interval_1n, k1, &w, true); - printTimeWindow(&w, calendar_interval_1n.precision, k1); + printTimeWindow(&w, precision, k1); printf("next\n"); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); getNextTimeWindow(&calendar_interval_1n, &w, TSDB_ORDER_ASC); - printTimeWindow(&w, calendar_interval_1n.precision, key); + printTimeWindow(&w, precision, key); + printf("----------------interval_1d_clendar_sliding_1n---------\n"); + SInterval interval_1d_calendar_sliding_1n = createInterval(1*86400*1000L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); - printf("----------------------------------------------------------\n"); - SInterval calendar_interval_1d_sliding_1n = createInterval(1*86400*100L, 1, 0, 'd', 'n', 'd', TSDB_TIME_PRECISION_MILLI); + k1 = 1664409600 * 1000L; + getInitialStartTimeWindow(&interval_1d_calendar_sliding_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + + printf("next time window:\n"); + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + getNextTimeWindow(&interval_1d_calendar_sliding_1n, &w, TSDB_ORDER_ASC); + printTimeWindow(&w, precision, k1); + + printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n"); + SInterval offset_1n = createInterval(1*86400*1000L, 1*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); } From a488f31ba98998cd338e9f21dd135999e78a332f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Feb 2023 09:52:11 +0800 Subject: [PATCH 004/106] refactor:do some internal refactor. --- source/common/src/ttime.c | 8 ++++---- source/libs/executor/test/timewindowTest.cpp | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index bbb58cc63d..de6a18f7c8 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -871,10 +871,10 @@ int64_t taosTimeTruncate(int64_t ts, const SInterval* pInterval) { // try to move current window to the left-hande-side, due to the offset effect. int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - int64_t newEnd = end; - while (newEnd >= ts) { - end = newEnd; - newEnd = taosTimeAdd(newEnd, -pInterval->sliding, pInterval->slidingUnit, precision); + int64_t newe = end; + while (newe >= ts) { + end = newe; + newe = taosTimeAdd(newe, -pInterval->sliding, pInterval->slidingUnit, precision); } start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1; diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index be5e217ed1..2894c66587 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -151,7 +151,10 @@ TEST(testCase, timewindow_gen) { printTimeWindow(&w, precision, k1); printf("----------------interval_1d_sliding_1d_calendar_offset_1n---------\n"); - SInterval offset_1n = createInterval(1*86400*1000L, 1*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); + SInterval offset_1n = createInterval(10*86400*1000L, 10*86400*1000L, 1, 'd', 'd', 'n', TSDB_TIME_PRECISION_MILLI); + getInitialStartTimeWindow(&offset_1n, k1, &w, true); + printTimeWindow(&w, precision, k1); + } From 2400104ec80fc98ac91a8c8d8253374565601cfe Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 11 May 2023 14:26:42 +0800 Subject: [PATCH 005/106] test: init add eco-system --- tests/system-test/eco-system/main.py | 37 +++ .../eco-system/manager/drop_table.py | 146 +++++++++++ .../eco-system/manager/same_column.py | 181 +++++++++++++ .../eco-system/manager/schema_change.py | 239 ++++++++++++++++++ .../eco-system/schemaless/insert.py | 147 +++++++++++ 5 files changed, 750 insertions(+) create mode 100644 tests/system-test/eco-system/main.py create mode 100644 tests/system-test/eco-system/manager/drop_table.py create mode 100644 tests/system-test/eco-system/manager/same_column.py create mode 100644 tests/system-test/eco-system/manager/schema_change.py create mode 100644 tests/system-test/eco-system/schemaless/insert.py diff --git a/tests/system-test/eco-system/main.py b/tests/system-test/eco-system/main.py new file mode 100644 index 0000000000..a1f72147a0 --- /dev/null +++ b/tests/system-test/eco-system/main.py @@ -0,0 +1,37 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + def run(self): + tdLog.info(" ------ eco-system main -------") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/eco-system/manager/drop_table.py b/tests/system-test/eco-system/manager/drop_table.py new file mode 100644 index 0000000000..d4cdf4a541 --- /dev/null +++ b/tests/system-test/eco-system/manager/drop_table.py @@ -0,0 +1,146 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(0, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + # show tables + def show_tables(self): + sql = "show tables;" + row_cnt = tdSql.query(sql) + tables = [] + for i in range(0, row_cnt): + tb_name = tdSql.queryResult[i][0] + tables.append(tb_name) + + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def drop_table(self, change_cnt): + # init + + tables = self.show_tables() + + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.drop_table(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/same_column.py b/tests/system-test/eco-system/manager/same_column.py new file mode 100644 index 0000000000..beaf4e449e --- /dev/null +++ b/tests/system-test/eco-system/manager/same_column.py @@ -0,0 +1,181 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'bc': 'bool', + 'fc': 'float', + 'dc': 'double', + 'ti': 'tinyint', + 'si': 'smallint', + 'ic': 'int', + 'bi': 'bigint', + 'uit': 'tinyint unsigned', + 'usi': 'smallint unsigned', + 'ui': 'int unsigned', + 'ubi': 'bigint unsigned', + 'bin': 'binary(32)', + 'nch': 'nchar(64)' + } + self.tag_dict = { + 'groupid': 'tinyint', + 'location': 'binary(16)', + 'tfc': 'float', + 'tdc': 'double', + 'tti': 'tinyint', + 'tsi': 'smallint', + 'tic': 'int', + 'tbi': 'bigint', + 'tuit': 'tinyint unsigned', + 'tusi': 'smallint unsigned', + 'tui': 'int unsigned', + 'tubi': 'bigint unsigned', + 'tbin': 'binary(32)', + 'tnch': 'nchar(64)' + } + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # execute sql + def execute(self, sql): + try: + tdSql.execute(sql, 3) + tdLog.info(f" exec ok. {sql}") + except: + tdLog.info(f" exe failed. {sql}") + traceback.print_exc() + + + # query + def query_table(self, columns, tags): + if len(columns) < 5 : + return + if len(tags) < 5: + return + + sel_cols = random.sample(columns, random.randint(1,int(len(columns)-1))) + sel_tags = random.sample(tags, random.randint(1, int(len(tags)-1))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_columns(self, change_cnt): + # init + + ncol = len(self.column_dict) + ntag = len(self.tag_dict) + + for i in range(change_cnt): + col_idx = random.randint(0, ncol - 1) + tag_idx = random.randint(0, ntag - 1) + + cols = list(self.column_dict.keys()) + tags = list(self.tag_dict.keys()) + + # column + key = cols[col_idx] + value = self.column_dict[key] + sql = f'alter table meters drop column {key}' + self.execute(sql) + sql = f'alter table meters add column {key} {value}' + self.execute(sql) + + + # column + key = tags[col_idx] + value = self.tag_dict[key] + sql = f'alter table meters drop tag {key}' + self.execute(sql) + sql = f'alter table meters add tag {key} {value}' + self.execute(sql) + + # drop and rename + if i % 5 == 0: + # update columns + #columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(cols)} tags= {len(tags)}======") + self.query_table(cols, tags) + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_columns(1000000) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/manager/schema_change.py b/tests/system-test/eco-system/manager/schema_change.py new file mode 100644 index 0000000000..400d2b100b --- /dev/null +++ b/tests/system-test/eco-system/manager/schema_change.py @@ -0,0 +1,239 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.column_dict = { + 'col0': 'int', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'varchar(20)', + 'col13': 'nchar(20)' + } + self.tag_dict = { + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': 'varchar(20)', + 't13': 'nchar(20)', + 't14': 'timestamp' + } + + + # delete + def delete_col(self, columns, cnt, max_col): + # delte for random + for i in range(cnt): + col_cnt = len(columns) + if col_cnt == 0: + return + sel = random.randint(0, col_cnt - 1) + sql = f"alter table meters drop column `{columns[sel]}`" + try: + tdSql.execute(sql) + tdLog.info(f" drop cur col={len(columns)} max_col={max_col} {sql}") + del columns[sel] + except: + tdLog.info(f" drop column failed. {sql}") + traceback.print_exc() + + + # describe table + def describe_table(self, tbname): + columns = [] + tags = [] + sql = f"describe {tbname}" + row_cnt = tdSql.query(sql) + for i in range(20, row_cnt): + col_name = tdSql.queryResult[i][0] + type_name = tdSql.queryResult[i][3] + if type_name == "TAG": + tags.append(col_name) + else: + columns.append(col_name) + + return columns,tags + + def renames(self, tags, cnt): + col_cnt = len(tags) + if col_cnt < 10: + return + for i in range(cnt): + sel = random.randint(1, col_cnt-3) + new_name = tags[sel] + "n" + sql = f"alter table meters rename tag `{tags[sel]}` `{new_name}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + tags[sel] = new_name + except: + tdLog.info(f" rename tag failed. {sql}") + traceback.print_exc() + + + def drop_tag(self, tags, cnt): + for i in range(cnt): + tag_cnt = len(tags) + sel = random.randint(1, tag_cnt-1) + sql = f"alter table meters drop tag `{tags[sel]}` " + try: + tdSql.execute(sql) + tdLog.info(sql) + del tags[sel] + except: + tdLog.info(f" drop tags failed. {sql}") + traceback.print_exc() + + # query + def query_table(self, columns, tags): + if len(columns) < 10 : + return + if len(tags) < 10: + return + + sel_cols = random.sample(columns, random.randint(2,int(len(columns)/10))) + sel_tags = random.sample(tags, random.randint(1,int(len(tags)/10))) + + field_cols = ",".join(sel_cols) + field_tags = ",".join(sel_tags) + + #sql = f"select {field_cols},{field_tags} from meters ;" + sql = f"select {field_cols},{field_tags} from meters" + try: + tdLog.info( " query sql:" + sql) + tdSql.query("select * from meters limit 1") + except: + tdLog.info( " query failed :" + sql) + traceback.print_exc() + + # change table schema + def change_schema(self, change_cnt): + # init + columns, tags = self.describe_table("meters") + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + for i in range(change_cnt): + col_cnt = len(self.column_dict) + icol = random.randint(0, col_cnt-1) + key = f"col{icol}" + col_name = key + f"_{i}_{random.randint(1,100)}" + col_type = self.column_dict[key] + sql = f'alter table meters add column `{col_name}` {col_type}' + sql_tag = f'alter table meters add tag `t_{col_name}` {col_type}' + + try: + tdSql.execute(sql) + tdLog.info(f" add cur col={len(columns)} max_col={max_col} {sql}") + columns.append(col_name) + if random.randint(1, 4) == 2: + tdSql.execute(sql_tag) + tdLog.info(f" add tag tag_cnt={len(tags)} {sql_tag}") + + except: + tdLog.info(f" add column failed. {sql}") + traceback.print_exc() + + + col_cnt = len(columns) + # delete + if col_cnt > max_col + 100: + self.delete_col(columns, random.randint(1, 30), max_col) + elif col_cnt >= max_col + 30: + self.delete_col(columns, random.randint(1, 4), max_col) + max_col = random.randint(200, 2000) + tdLog.info(f" ----------- set max column = {max_col} -------------") + elif col_cnt > max_col: + self.delete_col(columns, random.randint(1, 3), max_col) + + + + if i % 50 == 0: + sql = f"flush database {self.dbname};" + tdSql.execute(sql) + tdLog.info(f" ***** {sql} *****") + + # query + if i % 70 == 0: + self.query_table(columns, tags) + + # drop and rename + if i % 10 == 0: + # update columns + columns,tags = self.describe_table("meters") + tdLog.info(f" ======= describe table column count = {len(columns)} tags= {len(tags)}======") + + if random.randint(1,3) == 2: + self.query_table(columns, tags) + + if len(tags) > 50: + self.drop_tag(tags, random.randint(1, 30)) + + self.renames(tags, random.randint(1, 10)) + + + # sleep + #time.sleep(0.3) + + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + + # change meters + self.change_schema(1000000) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/eco-system/schemaless/insert.py b/tests/system-test/eco-system/schemaless/insert.py new file mode 100644 index 0000000000..def70748d7 --- /dev/null +++ b/tests/system-test/eco-system/schemaless/insert.py @@ -0,0 +1,147 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random +import time +import traceback +import taos +import string +from taos import schemaless + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.setsql = TDSetSql() + self.conn = conn + self.schema = {} + + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + def genCol(self, col_name, isTag): + col_types = ["str","f64","f32","i8","u8","i16","u16","i32","u32","i64","u64"] + if self.schema.get(col_name) == None: + col_type = random.choice(col_types) + self.schema[col_name] = col_type + else: + col_type = self.schema[col_name] + + is_num = True + val = "" + if col_type == "str": + val = self.random_string(random.randint(1, 10)) + is_num = False + elif col_type == "f64": + val = random.randrange(-100000000000000, 1000000000000)/3*2.25678 + elif col_type == "f32": + val = random.randrange(-100000000, 1000000000)/3*1.2345 + elif col_type == "i8": + val = random.randint(-128, 127) + elif col_type == "u8": + val = random.randint(0, 256) + elif col_type == "i16": + val = random.randint(-32768, 32767) + elif col_type == "u16": + val = random.randint(0, 256*256) + elif col_type == "i32": + val = random.randint(-256*256*256*128, 256*256*256*128) + elif col_type == "u32": + val = random.randint(0, 256*256*256*256) + elif col_type == "i64": + val = random.randint(-256*256*256*256*256*256*256*128, 256*256*256*256*256*256*256*128) + elif col_type == "u64": + val = random.randint(0, 256*256*256*256*256*256*256*256) + else: + val = 100 + + if isTag: + col_val = val + elif is_num: + col_val = f'{val}{col_type}' + else: + col_val = '"' + val + '"' + + return f'{col_name}={col_val}' + + + # cols + def genCols(self, pre, max, index, isTag): + col_cnt = random.randint(1, max) + cols = [] + for i in range(col_cnt): + col_name = f'{pre}_{index}_{i}' + cols.append(self.genCol(col_name, isTag)) + + return ",".join(cols) + + + # execute sql + def insert(self,sql,i): + print("schema less insert") + try: + self.conn.schemaless_insert([sql], schemaless.SmlProtocol.LINE_PROTOCOL, schemaless.SmlPrecision.MILLI_SECONDS) + tdLog.info(f" exec ok i={i} {sql}") + except: + tdLog.info(f" exe failed. i={i} {sql}") + traceback.print_exc() + + + # change table schema + def schemaless_insert(self, change_cnt): + # init + ts = 1683194263000 + for i in range(change_cnt): + t1 = i % 1000 + index = int(i/10000) % 600 + cols = self.genCols("c", 5, index, False) + tags = f"t1={t1},t2=abc,t3=work" + sql = f'sta,{tags} {cols} {ts + i}' + self.insert(sql, i) + + # run + def run(self): + # seed + #random.seed(int(time.time())) + self.dbname = "schema_change" + + # switch db + tdSql.execute(f"use {self.dbname};") + tdSql.execute(f"drop table if exists sta;") + + + + # change meters + try: + self.schemaless_insert(1000) + except: + traceback.print_exc() + + print(self.schema) + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 629ab2b85fb59eeebc3e4634de1db0c9b0b91586 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 15 May 2023 16:10:11 +0800 Subject: [PATCH 006/106] fix: extract tag equal condition --- include/libs/nodes/plannodes.h | 2 + source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 18 ++++-- source/libs/nodes/src/nodesMsgFuncs.c | 10 ++- source/libs/nodes/src/nodesUtilFuncs.c | 2 + source/libs/planner/src/planOptimizer.c | 73 ++++++++++++++++++++++ source/libs/planner/src/planPhysiCreater.c | 3 + 7 files changed, 103 insertions(+), 6 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index ad4b59714c..0e235191b1 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,6 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; + SNode* pTagEqualConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -405,6 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; + SNode* pTagEqualCondtions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index d9a4c5178f..91db527a02 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,6 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); + CLONE_NODE_FIELD(pTagEqualConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0aeb83ce08..b4e2d95e26 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1416,6 +1416,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanMergeCondition = "MergeConditions"; +static const char* jkJoinLogicPlanTagEqualConditions = "TagEqualConditions"; static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; @@ -1430,7 +1431,9 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pTagEqualConditions); + } return code; } @@ -1447,7 +1450,9 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pTagEqualConditions); + } return code; } @@ -1878,6 +1883,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition"; static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; +static const char* jkJoinPhysiPlanTagEqualConditions = "TagEqualConditions"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; @@ -1898,7 +1904,9 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pTagEqualCondtions); + } return code; } @@ -1921,7 +1929,9 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } - + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pTagEqualCondtions); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 6c6b6c0e81..45cebb4559 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2317,7 +2317,8 @@ enum { PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, PHY_SORT_MERGE_JOIN_CODE_TARGETS, - PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER + PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, + PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS }; static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2339,7 +2340,9 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pTagEqualCondtions); + } return code; } @@ -2368,6 +2371,9 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER: code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; + case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagEqualCondtions); + break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f71eef7969..d97b17e30c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1072,6 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); + nodesDestroyNode(pLogicNode->pTagEqualConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1204,6 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); + nodesDestroyNode(pPhyNode->pTagEqualCondtions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 66d85a9c89..f5ef02e207 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -740,6 +740,75 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin return code; } +static bool pushDownCondOptIsTag(SNode* pNode, SNodeList* pTableCols) { + if (QUERY_NODE_COLUMN != nodeType(pNode)) { + return false; + } + SColumnNode* pCol = (SColumnNode*)pNode; + if (COLUMN_TYPE_TAG != pCol->colType) { + return false; + } + return pushDownCondOptBelongThisTable(pNode, pTableCols); +} + +static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { + if (QUERY_NODE_OPERATOR != nodeType(pCond)) { + return false; + } + SOperatorNode* pOper = (SOperatorNode*)pCond; + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + if (pushDownCondOptIsTag(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsTag(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsTag(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsTag(pOper->pRight, pLeftCols); + } + return false; +} + +static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); + + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pTagEqualConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (pushDownCondOptIsTagEqualCond(pJoin, pCond)) { + code = nodesListMakeAppend(&pTagEqualConds, nodesCloneNode(pCond)); + } + } + + SNode* pTempTagEqCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempTagEqCond, &pTagEqualConds); + } + + if (TSDB_CODE_SUCCESS == code) { + pJoin->pTagEqualConditions = pTempTagEqCond; + return TSDB_CODE_SUCCESS; + } else { + nodesDestroyList(pTagEqualConds); + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t pushDownCondOptJoinExtractTagEqualCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { + return pushDownCondOptJoinExtractTagEqualLogicCond(pJoin); + } + + if (pushDownCondOptIsTagEqualCond(pJoin, pJoin->pOnConditions)) { + pJoin->pTagEqualConditions = nodesCloneNode(pJoin->pOnConditions); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { return TSDB_CODE_SUCCESS; @@ -774,6 +843,10 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p code = pushDownCondOptJoinExtractMergeCond(pCxt, pJoin); } + if (TSDB_CODE_SUCCESS == code) { + code = pushDownCondOptJoinExtractTagEqualCond(pCxt, pJoin); + } + if (TSDB_CODE_SUCCESS == code) { OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); pCxt->optimized = true; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e2c2e4c655..9c6c227480 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,6 +705,9 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pTagEqualConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pTagEqualConditions, &pJoin->pTagEqualCondtions); + } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); } From 789b0c7178e5ce21ea5e4231952066d9562c6023 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 16 May 2023 15:42:30 +0800 Subject: [PATCH 007/106] fix: select , from ... return error data when strlen(expr1) > TSDB_COL_NAME_LEN --- source/libs/nodes/src/nodesToSQLFuncs.c | 4 +-- source/libs/nodes/src/nodesUtilFuncs.c | 2 +- source/libs/parser/src/parAstCreater.c | 15 +++++++++-- source/libs/parser/src/parTranslater.c | 4 ++- source/libs/planner/src/planLogicCreater.c | 1 + tests/parallel_test/cases.task | 1 + tests/script/tsim/query/bug3398.sim | 30 ++++++++++++++++++++++ 7 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 tests/script/tsim/query/bug3398.sim diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 0181da92a9..b57bba0cc9 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -120,9 +120,9 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } if (colNode->tableAlias[0]) { - *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "`%s`", colNode->node.userAlias); } else { - *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->colName); + *len += snprintf(buf + *len, bufSize - *len, "%s", colNode->node.userAlias); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index f71eef7969..acd028d8ae 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2054,7 +2054,7 @@ char* nodesGetNameFromColumnNode(SNode* pNode) { return "NULL"; } - return ((SColumnNode*)pNode)->colName; + return ((SColumnNode*)pNode)->node.userAlias; } int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots) { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 5a47ed731d..6f866c027e 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -259,8 +259,19 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { strcpy(pExpr->userAlias, ((SColumnNode*)pExpr)->colName); } else { int32_t len = TMIN(sizeof(pExpr->aliasName) - 1, pRawExpr->n); - strncpy(pExpr->aliasName, pRawExpr->p, len); - pExpr->aliasName[len] = '\0'; + + // See TS-3398. + // Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN]. + // If aliasName is truncated, hash value of aliasName could be the same. + T_MD5_CTX ctx; + tMD5Init(&ctx); + tMD5Update(&ctx, (uint8_t*)pRawExpr->p, pRawExpr->n); + tMD5Final(&ctx); + char* p = pExpr->aliasName; + for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { + sprintf(p, "%02x", ctx.digest[i]); + p += 2; + } strncpy(pExpr->userAlias, pRawExpr->p, len); pExpr->userAlias[len] = '\0'; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f4c86d4849..9a45add98f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -827,7 +827,7 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum strcpy(pCol->node.aliasName, pCol->colName); } if ('\0' == pCol->node.userAlias[0]) { - strcpy(pCol->node.userAlias, pCol->colName); + strcpy(pCol->node.userAlias, pExpr->userAlias); } pCol->node.resType = pExpr->resType; } @@ -1760,6 +1760,7 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pVal->node.aliasName, ((SExprNode*)*pNode)->aliasName); + strcpy(pVal->node.userAlias, ((SExprNode*)*pNode)->userAlias); pVal->node.resType = ((SExprNode*)*pNode)->resType; if (NULL == pLiteral) { pVal->isNull = true; @@ -2739,6 +2740,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { } else { len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->aliasName); strncpy(pFunc->node.aliasName, buf, TMIN(len, sizeof(pFunc->node.aliasName) - 1)); + len = snprintf(buf, sizeof(buf), "%s(%s)", pSrcFunc->functionName, pExpr->userAlias); strncpy(pFunc->node.userAlias, buf, TMIN(len, sizeof(pFunc->node.userAlias) - 1)); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 39783868b3..b9ea5ba0cd 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -100,6 +100,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SExprNode* pToBeRewrittenExpr = (SExprNode*)(*pNode); pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); + strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { setColumnInfo((SFunctionNode*)pExpr, pCol); diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4065ac5bee..484ac82acd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -904,6 +904,7 @@ ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/nullColSma.sim +,,y,script,./test.sh -f tsim/query/bug3398.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim ,,y,script,./test.sh -f tsim/mnode/basic1.sim diff --git a/tests/script/tsim/query/bug3398.sim b/tests/script/tsim/query/bug3398.sim new file mode 100644 index 0000000000..3ca88cf459 --- /dev/null +++ b/tests/script/tsim/query/bug3398.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database +sql create database test + +print =============== create super table and child table +sql use test + +sql CREATE STABLE st (day timestamp, c2 int) TAGS (vin binary(32)) + +sql insert into test.g using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.t using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) +sql insert into test.tg using st TAGS ("TAG1") values("2023-05-03 00:00:00.000", 1) + +sql select sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end), sum(case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end + case when t.c2 is NULL then 0 else 1 end) from test.t t, test.g g, test.tg tg where t.day = g.day and t.day = tg.day and t.day between '2021-05-03' and '2023-05-04' and t.vin = 'TAG1' and t.vin = g.vin and t.vin = tg.vin group by t.day; + +print $rows $data00 $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi + +if $data01 != 3.000000000 then + return -1 +endi From c6555c3b3778808e910a2044714913cd62895d61 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 16 May 2023 16:25:55 +0800 Subject: [PATCH 008/106] fix: join operator finished --- source/libs/executor/src/joinoperator.c | 369 +++++++++++++++++++----- source/libs/planner/src/planOptimizer.c | 8 + 2 files changed, 307 insertions(+), 70 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 754b5f4737..a5591b22b3 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -30,11 +30,11 @@ typedef struct SJoinRowCtx { bool rowRemains; int64_t ts; SArray* leftRowLocations; - SArray* rightRowLocations; SArray* leftCreatedBlocks; SArray* rightCreatedBlocks; int32_t leftRowIdx; int32_t rightRowIdx; + SSHashObj* buildTableTSRange; } SJoinRowCtx; typedef struct SJoinOperatorInfo { @@ -50,6 +50,17 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; + SNode* pTagEqualConditions; + + SArray* leftTagCols; + SArray* leftTagKeys; + char* leftTagKeyBuf; + int32_t leftTagKeyLen; + + SArray* rightTagCols; + SArray* rightTagKeys; + char* rightTagKeyBuf; + int32_t rightTagKeyLen; SJoinRowCtx rowCtx; } SJoinOperatorInfo; @@ -92,6 +103,147 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } +static void extractTagEqualColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, + SColumn* pLeft, SColumn* pRight) { + SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft; + SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight; + if (pLeftNode->dataBlockId == pRightNode->dataBlockId || pLeftNode->dataBlockId == pDownstreams[0]->resultDataBlockId) { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + } else { + *pLeft = extractColumnFromColumnNode((SColumnNode*)pOperNode->pRight); + *pRight = extractColumnFromColumnNode((SColumnNode*)pOperNode->pLeft); + } +} + +static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pTagEqualNode, + SArray* leftTagEqCols, SArray* rightTagEqCols) { + SColumn left = {0}; + SColumn right = {0}; + if (nodeType(pTagEqualNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pTagEqualNode)->condType == LOGIC_COND_TYPE_AND) { + SNode* pNode = NULL; + FOREACH(pNode, ((SLogicConditionNode*)pTagEqualNode)->pParameterList) { + SOperatorNode* pOperNode = (SOperatorNode*)pNode; + extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } + return; + } + + if (nodeType(pTagEqualNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pOperNode = (SOperatorNode*)pTagEqualNode; + extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + taosArrayPush(leftTagEqCols, &left); + taosArrayPush(rightTagEqCols, &right); + } +} + +static int32_t initTagColsGroupkeys(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { + *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); + if ((*pGroupColVals) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); + (*keyLen) += pCol->bytes; // actual data + null_flag + + SGroupKeys key = {0}; + key.bytes = pCol->bytes; + key.type = pCol->type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pCol->bytes); + if (key.pData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush((*pGroupColVals), &key); + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + (*keyLen) += nullFlagSize; + + (*keyBuf) = taosMemoryCalloc(1, (*keyLen)); + if ((*keyBuf) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static void fillGroupKeyValsFromTagCols(SArray* pCols, SArray* pGroupKeys, SSDataBlock* pBlock, int32_t rowIndex) { + SColumnDataAgg* pColAgg = NULL; + + size_t numOfGroupCols = taosArrayGetSize(pCols); + + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SColumn* pCol = (SColumn*) taosArrayGet(pCols, i); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + + // valid range check. todo: return error code. + if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) { + continue; + } + + if (pBlock->pBlockAgg != NULL) { + pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + } + + SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + pkey->isNull = true; + } else { + pkey->isNull = false; + char* val = colDataGetData(pColInfoData, rowIndex); + if (pkey->type == TSDB_DATA_TYPE_JSON) { + if (tTagIsJson(val)) { + terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; + return; + } + int32_t dataLen = getJsonValueLen(val); + memcpy(pkey->pData, val, dataLen); + } else if (IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, varDataTLen(val)); + ASSERT(varDataTLen(val) <= pkey->bytes); + } else { + memcpy(pkey->pData, val, pkey->bytes); + } + } + } +} + +static int32_t combineGroupKeysIntoBuf(void* pKey, const SArray* pGroupKeys) { + size_t numOfGroupCols = taosArrayGetSize(pGroupKeys); + + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; + for (int32_t i = 0; i < numOfGroupCols; ++i) { + SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); + if (pkey->isNull) { + isNull[i] = 1; + continue; + } + + isNull[i] = 0; + if (pkey->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(pkey->pData); + memcpy(pStart, (pkey->pData), dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pkey->type)) { + varDataCopy(pStart, pkey->pData); + pStart += varDataTLen(pkey->pData); + ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); + } else { + memcpy(pStart, pkey->pData, pkey->bytes); + pStart += pkey->bytes; + } + } + + return (int32_t)(pStart - (char*)pKey); +} + SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); @@ -153,6 +305,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } + pInfo->pTagEqualConditions = pJoinNode->pTagEqualCondtions; + pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); + extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); + initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); + initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -179,8 +338,22 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } +static void freeGroupKeyData(void* param) { + SGroupKeys* pKey = (SGroupKeys*)param; + taosMemoryFree(pKey->pData); +} + void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; + + taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->rightTagCols); + + taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->leftTagCols); + nodesDestroyNode(pJoinOperator->pCondAfterMerge); pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); @@ -300,22 +473,121 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } +static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj** ppHashObj) { + int32_t buildTableCap = MIN(taosArrayGetSize(rightRowLocations), 4096); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); + for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { + SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); + fillGroupKeyValsFromTagCols(pInfo->rightTagCols, pInfo->rightTagKeys, rightRow->pDataBlock, rightRow->pos); + int32_t keyLen = combineGroupKeysIntoBuf(pInfo->rightTagKeyBuf, pInfo->rightTagKeys); + SArray** ppRows = tSimpleHashGet(buildTable, pInfo->rightTagKeyBuf, keyLen); + if (!ppRows) { + SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); + taosArrayPush(rows, rightRow); + tSimpleHashPut(buildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); + } else { + taosArrayPush(*ppRows, rightRow); + } + } + *ppHashObj = buildTable; + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, + const SArray* leftRowLocations, int32_t leftRowIdx, + int32_t rightRowIdx, SSHashObj* rightTableHash, bool* pReachThreshold) { + *pReachThreshold = false; + uint32_t limitRowNum = pOperator->resultInfo.threshold; + SJoinOperatorInfo* pJoinInfo = pOperator->info; + size_t leftNumJoin = taosArrayGetSize(leftRowLocations); + + int32_t i,j; + + for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { + SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); + fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); + int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); + SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); + if (!ppRightRows) { + continue; + } + SArray* pRightRows = *ppRightRows; + size_t rightRowsSize = taosArrayGetSize(pRightRows); + for (j = rightRowIdx; j < rightRowsSize; ++j) { + if (*nRows >= limitRowNum) { + *pReachThreshold = true; + break; + } + + SRowLocation* rightRow = taosArrayGet(pRightRows, j); + mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, + rightRow->pos); + ++*nRows; + } + if (*pReachThreshold) { + break; + } + } + + if (*pReachThreshold) { + pJoinInfo->rowCtx.rowRemains = true; + pJoinInfo->rowCtx.leftRowIdx = i; + pJoinInfo->rowCtx.rightRowIdx = j; + } + return TSDB_CODE_SUCCESS; +} + +static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + + tSimpleHashCleanup(pBuildTable); +} + +static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, + SArray* rightCreatedBlocks, SSHashObj* rightTableHash) { + for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); + blockDataDestroy(pBlock); + } + taosArrayDestroy(rightCreatedBlocks); + for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { + SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); + blockDataDestroy(pBlock); + } + mergeJoinDestoryBuildTable(rightTableHash); + + taosArrayDestroy(leftCreatedBlocks); + taosArrayDestroy(leftRowLocations); + + pJoinInfo->rowCtx.rowRemains = false; + pJoinInfo->rowCtx.leftRowLocations = NULL; + pJoinInfo->rowCtx.leftCreatedBlocks = NULL; + pJoinInfo->rowCtx.rightCreatedBlocks = NULL; + pJoinInfo->rowCtx.buildTableTSRange = NULL; +} + static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, int32_t* nRows) { int32_t code = TSDB_CODE_SUCCESS; SJoinOperatorInfo* pJoinInfo = pOperator->info; SArray* leftRowLocations = NULL; SArray* leftCreatedBlocks = NULL; - SArray* rightRowLocations = NULL; SArray* rightCreatedBlocks = NULL; int32_t leftRowIdx = 0; int32_t rightRowIdx = 0; - int32_t i, j; - + SSHashObj* rightTableHash = NULL; + if (pJoinInfo->rowCtx.rowRemains) { leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; - rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; + rightTableHash = pJoinInfo->rowCtx.buildTableTSRange; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; rightRowIdx = pJoinInfo->rowCtx.rightRowIdx; @@ -323,85 +595,42 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t leftRowLocations = taosArrayInit(8, sizeof(SRowLocation)); leftCreatedBlocks = taosArrayInit(8, POINTER_BYTES); - rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); + SArray* rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); rightCreatedBlocks = taosArrayInit(8, POINTER_BYTES); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 0, pJoinInfo->leftCol.slotId, pJoinInfo->pLeft, pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); + mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); + taosArrayDestroy(rightRowLocations); } size_t leftNumJoin = taosArrayGetSize(leftRowLocations); - size_t rightNumJoin = taosArrayGetSize(rightRowLocations); - uint32_t maxRowNum = *nRows + (leftNumJoin - leftRowIdx - 1) * rightNumJoin + rightNumJoin - rightRowIdx; - uint32_t limitRowNum = maxRowNum; - if (maxRowNum > pOperator->resultInfo.threshold) { - limitRowNum = pOperator->resultInfo.threshold; - if (!pJoinInfo->rowCtx.rowRemains) { + code = blockDataEnsureCapacity(pRes, pOperator->resultInfo.threshold); + if (code != TSDB_CODE_SUCCESS) { + qError("%s can not ensure block capacity for join. left: %zu", GET_TASKID(pOperator->pTaskInfo), + leftNumJoin); + } + + bool reachThreshold = false; + + if (code == TSDB_CODE_SUCCESS) { + mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, + rightRowIdx, rightTableHash, &reachThreshold); + } + + if (!reachThreshold) { + mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, + rightTableHash); + + } else { pJoinInfo->rowCtx.rowRemains = true; pJoinInfo->rowCtx.ts = timestamp; pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; - pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - } - } - - code = blockDataEnsureCapacity(pRes, limitRowNum); - if (code != TSDB_CODE_SUCCESS) { - qError("%s can not ensure block capacity for join. left: %zu, right: %zu", GET_TASKID(pOperator->pTaskInfo), - leftNumJoin, rightNumJoin); - } - - - if (code == TSDB_CODE_SUCCESS) { - bool done = false; - for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { - for (j = rightRowIdx; j < rightNumJoin; ++j) { - if (*nRows >= limitRowNum) { - done = true; - break; - } - - SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); - SRowLocation* rightRow = taosArrayGet(rightRowLocations, j); - mergeJoinJoinLeftRight(pOperator, pRes, *nRows, leftRow->pDataBlock, leftRow->pos, rightRow->pDataBlock, - rightRow->pos); - ++*nRows; - } - if (done) { - break; - } - } - - if (maxRowNum > pOperator->resultInfo.threshold) { - pJoinInfo->rowCtx.leftRowIdx = i; - pJoinInfo->rowCtx.rightRowIdx = j; - } - } - - if (maxRowNum <= pOperator->resultInfo.threshold) { - for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(rightCreatedBlocks); - taosArrayDestroy(rightRowLocations); - for (int i = 0; i < taosArrayGetSize(leftCreatedBlocks); ++i) { - SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); - blockDataDestroy(pBlock); - } - taosArrayDestroy(leftCreatedBlocks); - taosArrayDestroy(leftRowLocations); - - if (pJoinInfo->rowCtx.rowRemains) { - pJoinInfo->rowCtx.rowRemains = false; - pJoinInfo->rowCtx.leftRowLocations = NULL; - pJoinInfo->rowCtx.rightRowLocations = NULL; - pJoinInfo->rowCtx.leftCreatedBlocks = NULL; - pJoinInfo->rowCtx.rightCreatedBlocks = NULL; - } + pJoinInfo->rowCtx.buildTableTSRange = rightTableHash; } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index f5ef02e207..8c111133fc 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -759,6 +759,14 @@ static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { if (OP_TYPE_EQUAL != pOper->opType) { return false; } + if (QUERY_NODE_COLUMN != nodeType(pOper->pLeft) || QUERY_NODE_COLUMN != nodeType(pOper->pRight)) { + return false; + } + SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft); + SColumnNode* pRight = (SColumnNode*)(pOper->pRight); + if (pLeft->node.resType.type != pRight->node.resType.type) { + return false; + } SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; if (pushDownCondOptIsTag(pOper->pLeft, pLeftCols)) { From a51de5cc9384b22d09ec27f9607f46361e548975 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 16 May 2023 16:31:58 +0800 Subject: [PATCH 009/106] fix: add restriction --- source/libs/planner/src/planOptimizer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8c111133fc..d76c25fc55 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -764,7 +764,8 @@ static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { } SColumnNode* pLeft = (SColumnNode*)(pOper->pLeft); SColumnNode* pRight = (SColumnNode*)(pOper->pRight); - if (pLeft->node.resType.type != pRight->node.resType.type) { + //TODO: add cast to operator and remove this restriction of optimization + if (pLeft->node.resType.type != pRight->node.resType.type || pLeft->node.resType.bytes != pRight->node.resType.bytes) { return false; } SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; From 568725a245d8e1a2f80dc056c85955bdd8a382cd Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 17 May 2023 08:33:47 +0800 Subject: [PATCH 010/106] fix: using right rows to build hash table when the tag equal cond is not null and same ts row row number is greater than 16 --- source/libs/executor/src/joinoperator.c | 74 ++++++++++++++++--------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index a5591b22b3..54ddab9e1e 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -34,7 +34,9 @@ typedef struct SJoinRowCtx { SArray* rightCreatedBlocks; int32_t leftRowIdx; int32_t rightRowIdx; + SSHashObj* buildTableTSRange; + SArray* rightRowLocations; } SJoinRowCtx; typedef struct SJoinOperatorInfo { @@ -306,12 +308,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t } pInfo->pTagEqualConditions = pJoinNode->pTagEqualCondtions; - pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); - pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); - extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); - initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); - initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); - + if (pInfo->pTagEqualConditions != NULL) { + pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); + extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); + initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); + initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { @@ -345,15 +348,15 @@ static void freeGroupKeyData(void* param) { void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; + if (pJoinOperator->pTagEqualConditions != NULL) { + taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->rightTagCols); - taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); - taosArrayDestroy(pJoinOperator->rightTagCols); - - taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); - taosArrayDestroy(pJoinOperator->leftTagCols); - + taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); + taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); + taosArrayDestroy(pJoinOperator->leftTagCols); + } nodesDestroyNode(pJoinOperator->pCondAfterMerge); pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); @@ -496,7 +499,7 @@ static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* right static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, const SArray* leftRowLocations, int32_t leftRowIdx, - int32_t rightRowIdx, SSHashObj* rightTableHash, bool* pReachThreshold) { + int32_t rightRowIdx, SSHashObj* rightTableHash, SArray* rightRowLocations, bool* pReachThreshold) { *pReachThreshold = false; uint32_t limitRowNum = pOperator->resultInfo.threshold; SJoinOperatorInfo* pJoinInfo = pOperator->info; @@ -506,13 +509,18 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); - fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); - int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); - SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); - if (!ppRightRows) { - continue; + SArray* pRightRows = NULL; + if (rightTableHash != NULL) { + fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); + int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); + SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); + if (!ppRightRows) { + continue; + } + pRightRows = *ppRightRows; + } else { + pRightRows = rightRowLocations; } - SArray* pRightRows = *ppRightRows; size_t rightRowsSize = taosArrayGetSize(pRightRows); for (j = rightRowIdx; j < rightRowsSize; ++j) { if (*nRows >= limitRowNum) { @@ -551,7 +559,7 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { } static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, - SArray* rightCreatedBlocks, SSHashObj* rightTableHash) { + SArray* rightCreatedBlocks, SSHashObj* rightTableHash, SArray* rightRowLocations) { for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); blockDataDestroy(pBlock); @@ -561,7 +569,12 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef SSDataBlock* pBlock = taosArrayGetP(leftCreatedBlocks, i); blockDataDestroy(pBlock); } - mergeJoinDestoryBuildTable(rightTableHash); + if (rightRowLocations != NULL) { + taosArrayDestroy(rightRowLocations); + } + if (rightTableHash != NULL) { + mergeJoinDestoryBuildTable(rightTableHash); + } taosArrayDestroy(leftCreatedBlocks); taosArrayDestroy(leftRowLocations); @@ -571,6 +584,7 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef pJoinInfo->rowCtx.leftCreatedBlocks = NULL; pJoinInfo->rowCtx.rightCreatedBlocks = NULL; pJoinInfo->rowCtx.buildTableTSRange = NULL; + pJoinInfo->rowCtx.rightRowLocations = NULL; } static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t timestamp, SSDataBlock* pRes, @@ -578,6 +592,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t int32_t code = TSDB_CODE_SUCCESS; SJoinOperatorInfo* pJoinInfo = pOperator->info; SArray* leftRowLocations = NULL; + SArray* rightRowLocations = NULL; SArray* leftCreatedBlocks = NULL; SArray* rightCreatedBlocks = NULL; int32_t leftRowIdx = 0; @@ -588,6 +603,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; rightTableHash = pJoinInfo->rowCtx.buildTableTSRange; + rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; rightRowIdx = pJoinInfo->rowCtx.rightRowIdx; @@ -602,8 +618,11 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); - mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); - taosArrayDestroy(rightRowLocations); + if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); + taosArrayDestroy(rightRowLocations); + rightRowLocations = NULL; + } } size_t leftNumJoin = taosArrayGetSize(leftRowLocations); @@ -617,12 +636,12 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t if (code == TSDB_CODE_SUCCESS) { mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, - rightRowIdx, rightTableHash, &reachThreshold); + rightRowIdx, rightTableHash, rightRowLocations, &reachThreshold); } if (!reachThreshold) { mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, - rightTableHash); + rightTableHash, rightRowLocations); } else { pJoinInfo->rowCtx.rowRemains = true; @@ -631,6 +650,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; pJoinInfo->rowCtx.buildTableTSRange = rightTableHash; + pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; } From 03e77288e8ca2b3ea9a7385c21f181593fa899a6 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 17 May 2023 10:07:19 +0800 Subject: [PATCH 011/106] fix: add on conditions null check and rightRowLocations no longer hides the outer scope rightRowLocations --- source/libs/executor/src/joinoperator.c | 4 ++-- source/libs/planner/src/planOptimizer.c | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 54ddab9e1e..07b8a4db11 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -477,7 +477,7 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator } static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj** ppHashObj) { - int32_t buildTableCap = MIN(taosArrayGetSize(rightRowLocations), 4096); + int32_t buildTableCap = TMIN(taosArrayGetSize(rightRowLocations), 256); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { @@ -611,7 +611,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t leftRowLocations = taosArrayInit(8, sizeof(SRowLocation)); leftCreatedBlocks = taosArrayInit(8, POINTER_BYTES); - SArray* rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); + rightRowLocations = taosArrayInit(8, sizeof(SRowLocation)); rightCreatedBlocks = taosArrayInit(8, POINTER_BYTES); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 0, pJoinInfo->leftCol.slotId, pJoinInfo->pLeft, diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d76c25fc55..5be67389c8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -806,6 +806,10 @@ static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin } static int32_t pushDownCondOptJoinExtractTagEqualCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (NULL == pJoin->pOnConditions) { + pJoin->pTagEqualConditions = NULL; + return TSDB_CODE_SUCCESS; + } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { return pushDownCondOptJoinExtractTagEqualLogicCond(pJoin); From 28b098c824a7e008f6317d85b91ea29d15159d0a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 May 2023 10:47:11 +0800 Subject: [PATCH 012/106] fix: union clause error message --- source/util/src/terror.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6f726c7eff..712c413e7c 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -372,7 +372,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many groups/time window in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") From 1ded7a052bfc5fd9e823236d35800c5dc1fd3996 Mon Sep 17 00:00:00 2001 From: Yu Sun Date: Wed, 17 May 2023 07:22:15 +0000 Subject: [PATCH 013/106] docs(connector-rust): update documents of connector for rust The connector for rust has been updated to use `deadpool` as default connection pool instead of `r2d2`, and the `r2d2` was only kept for blocking mode. --- docs/zh/08-connector/26-rust.mdx | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/zh/08-connector/26-rust.mdx b/docs/zh/08-connector/26-rust.mdx index eeb93a6558..5cf37f7946 100644 --- a/docs/zh/08-connector/26-rust.mdx +++ b/docs/zh/08-connector/26-rust.mdx @@ -64,6 +64,13 @@ taos = "*" taos = { version = "*", default-features = false, features = ["ws"] } ``` +当仅启用 `ws` 特性时，可同时指定 `r2d2` 使得在同步（blocking/sync）模式下使用 [r2d2] 作为连接池： + +```toml +[dependencies] +taos = { version = "*", default-features = false, features = ["r2d2", "ws"] } +``` + @@ -252,26 +259,24 @@ let conn: Taos = cfg.build(); ### 连接池 -在复杂应用中，建议启用连接池。[taos] 的连接池使用 [r2d2] 实现。 +在复杂应用中，建议启用连接池。[taos] 的连接池默认（异步模式）使用 [deadpool] 实现。 如下，可以生成一个默认参数的连接池。 ```rust -let pool = TaosBuilder::from_dsn(dsn)?.pool()?; +let pool: Pool = TaosBuilder::from_dsn("taos:///") + .unwrap() + .pool() + .unwrap(); ``` 同样可以使用连接池的构造器，对连接池参数进行设置： ```rust -let dsn = "taos://localhost:6030"; - -let opts = PoolBuilder::new() - .max_size(5000) // max connections - .max_lifetime(Some(Duration::from_secs(60 * 60))) // lifetime of each connection - .min_idle(Some(1000)) // minimal idle connections - .connection_timeout(Duration::from_secs(2)); - -let pool = TaosBuilder::from_dsn(dsn)?.with_pool_builder(opts)?; +let pool: Pool = Pool::builder(Manager::from_dsn(self.dsn.clone()).unwrap().0) + .max_size(88) // 最大连接数 + .build() + .unwrap(); ``` 在应用代码中，使用 `pool.get()?` 来获取一个连接对象 [Taos]。 @@ -511,6 +516,7 @@ consumer.unsubscribe().await; 其他相关结构体 API 使用说明请移步 Rust 文档托管网页：。 [taos]: https://github.com/taosdata/rust-connector-taos +[deadpool]: https://crates.io/crates/deadpool [r2d2]: https://crates.io/crates/r2d2 [TaosBuilder]: https://docs.rs/taos/latest/taos/struct.TaosBuilder.html [TaosCfg]: https://docs.rs/taos/latest/taos/struct.TaosCfg.html From ab7cb995a2eb87fd9c32168d1874cf99a3f07012 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 May 2023 15:31:53 +0800 Subject: [PATCH 014/106] fix: join push down condition issue --- source/libs/nodes/src/nodesEqualFuncs.c | 4 +--- source/libs/parser/test/parSelectTest.cpp | 2 ++ source/libs/planner/src/planPhysiCreater.c | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 156744ef1d..477d27d271 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -82,9 +82,7 @@ static bool columnNodeEqual(const SColumnNode* a, const SColumnNode* b) { COMPARE_STRING_FIELD(dbName); COMPARE_STRING_FIELD(tableName); COMPARE_STRING_FIELD(colName); - if (0 == a->tableId) { - COMPARE_STRING_FIELD(tableAlias); - } + COMPARE_STRING_FIELD(tableAlias); return true; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index ec6c69ea8d..c60257cc22 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -459,6 +459,8 @@ TEST_F(ParserSelectTest, joinSemanticCheck) { run("SELECT * FROM (SELECT tag1, SUM(c1) s FROM st1 GROUP BY tag1) t1, st1 t2 where t1.tag1 = t2.tag1", TSDB_CODE_PAR_NOT_SUPPORT_JOIN); + + run("SELECT count(*) FROM t1 a join t1 b on a.ts=b.ts where a.ts=b.ts"); } } // namespace ParserTest diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e2c2e4c655..651be08aec 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1643,6 +1643,9 @@ static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pChildren, (SNode*)pChild); } + if (TSDB_CODE_SUCCESS != code) { + break; + } } if (TSDB_CODE_SUCCESS == code) { From 73b3b3fc377be4480112e4f61f32f1959e79ac24 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 May 2023 15:48:12 +0800 Subject: [PATCH 015/106] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6724e2d303..eb4f04c914 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -291,13 +291,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - if (offset.val.type == TMQ_OFFSET__LOG) { - STqHandle* pHandle = taosHashGet(pTq->pHandle, offset.subKey, strlen(offset.subKey)); - if (pHandle && (walRefVer(pHandle->pRef, offset.val.version) < 0)) { - return -1; - } - } - return 0; } From 4bb25c280f0452c5b078580c1bba8b9b09a94b91 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 15:54:36 +0800 Subject: [PATCH 016/106] fix:add log --- source/dnode/vnode/src/tq/tq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9730ad2877..ff715f1828 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -359,7 +359,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } while (tqIsHandleExec(pHandle)) { - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", consumerId, vgId, req.subKey); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosMsleep(5); } @@ -372,6 +372,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } tqSetHandleExec(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); // 3. update the epoch value @@ -389,6 +390,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); tqSetHandleIdle(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; } @@ -555,7 +557,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg goto end; } else { while (tqIsHandleExec(pHandle)) { - tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); + tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); taosMsleep(5); } From 1d9dd153eacdb10e5e1d03d896ca104f1bcaf30b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 18:57:45 +0800 Subject: [PATCH 017/106] fix:error in pHandle lock --- source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/tq/tq.c | 64 +++++++++++++++------------- source/dnode/vnode/src/tq/tqMeta.c | 2 - source/dnode/vnode/src/tq/tqOffset.c | 2 - source/dnode/vnode/src/tq/tqRead.c | 2 - 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ef36b8429a..f0008f04cb 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -93,6 +93,7 @@ typedef struct { typedef enum tq_handle_status{ TMQ_HANDLE_STATUS_IDLE = 0, TMQ_HANDLE_STATUS_EXEC = 1, + TMQ_HANDLE_STATUS_DELETE = 2, }tq_handle_status; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ff715f1828..b1fca3faa8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -25,6 +25,7 @@ static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_EXEC;} static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_IDLE;} +static FORCE_INLINE void tqSetHandleDelete(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_DELETE;} int32_t tqInit() { int8_t old; @@ -297,13 +298,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } if (offset.val.type == TMQ_OFFSET__LOG) { - taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, offset.subKey, strlen(offset.subKey)); if (pHandle && (walSetRefVer(pHandle->pRef, offset.val.version) < 0)) { - taosWUnLockLatch(&pTq->lock); return -1; } - taosWUnLockLatch(&pTq->lock); } return 0; @@ -337,6 +335,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SMqPollReq req = {0}; + int code = 0; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); terrno = TSDB_CODE_INVALID_MSG; @@ -347,20 +346,29 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t reqEpoch = req.epoch; STqOffsetVal reqOffset = req.reqOffset; int32_t vgId = TD_VID(pTq->pVnode); + STqHandle* pHandle = NULL; - taosWLockLatch(&pTq->lock); - // 1. find handle - STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - if (pHandle == NULL) { - tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); - terrno = TSDB_CODE_INVALID_MSG; + while (1) { + taosWLockLatch(&pTq->lock); + // 1. find handle + pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle == NULL) { + tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; + taosWUnLockLatch(&pTq->lock); + return -1; + } + + bool exec = tqIsHandleExec(pHandle); + if(!exec) { + tqSetHandleExec(pHandle); + taosWUnLockLatch(&pTq->lock); + break; + } taosWUnLockLatch(&pTq->lock); - return -1; - } - while (tqIsHandleExec(pHandle)) { tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); - taosMsleep(5); + taosMsleep(10); } // 2. check re-balance status @@ -368,12 +376,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - taosWUnLockLatch(&pTq->lock); - return -1; + code = -1; + goto end; } - tqSetHandleExec(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); - taosWUnLockLatch(&pTq->lock); // 3. update the epoch value int32_t savedEpoch = pHandle->epoch; @@ -388,7 +395,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); - int code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); + code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); + +end: tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; @@ -405,8 +414,8 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { while (tqIsHandleExec(pHandle)) { - tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry", vgId, pHandle->subKey); - taosMsleep(5); + tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); + taosMsleep(10); } if (pHandle->pRef) { walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); @@ -473,7 +482,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); - taosWLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -556,20 +564,14 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { - while (tqIsHandleExec(pHandle)) { - tqDebug("sub req vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); - taosMsleep(5); - } - if (pHandle->consumerId == req.newConsumerId) { // do nothing tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); - atomic_add_fetch_32(&pHandle->epoch, 1); - +// atomic_add_fetch_32(&pHandle->epoch, 1); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); - atomic_store_32(&pHandle->epoch, 0); +// atomic_store_32(&pHandle->epoch, 0); } // kill executing task qTaskInfo_t pTaskInfo = pHandle->execHandle.task; @@ -580,13 +582,15 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg qStreamCloseTsdbReader(pTaskInfo); } // remove if it has been register in the push manager, and return one empty block to consumer + taosWLockLatch(&pTq->lock); tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } end: - taosWUnLockLatch(&pTq->lock); taosMemoryFree(req.qmsg); return ret; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 5654147b6d..b3d21887c9 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -352,9 +352,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); - taosWLockLatch(&pTq->lock); taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); - taosWUnLockLatch(&pTq->lock); } end: diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 377a5d1887..0a9905b544 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -78,7 +78,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // todo remove this if (offset.val.type == TMQ_OFFSET__LOG) { - taosWLockLatch(&pStore->pTq->lock); STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); if (pHandle) { if (walSetRefVer(pHandle->pRef, offset.val.version) < 0) { @@ -86,7 +85,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { // offset.val.version); } } - taosWUnLockLatch(&pStore->pTq->lock); } taosMemoryFree(pMemBuf); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 09574a6b62..a26a749af3 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1035,7 +1035,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t vgId = TD_VID(pTq->pVnode); // update the table list for each consumer handle - taosWLockLatch(&pTq->lock); while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); if (pIter == NULL) { @@ -1092,7 +1091,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } } - taosWUnLockLatch(&pTq->lock); // update the table list handle for each stream scanner/wal reader taosWLockLatch(&pTq->pStreamMeta->lock); From 405cdfa7bc5637be46072149303bed2a03e5b20c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 19:02:16 +0800 Subject: [PATCH 018/106] fix:error in pHandle lock --- source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/tq/tq.c | 1 - 2 files changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f0008f04cb..ef36b8429a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -93,7 +93,6 @@ typedef struct { typedef enum tq_handle_status{ TMQ_HANDLE_STATUS_IDLE = 0, TMQ_HANDLE_STATUS_EXEC = 1, - TMQ_HANDLE_STATUS_DELETE = 2, }tq_handle_status; typedef struct { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b1fca3faa8..d8ebf72549 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -25,7 +25,6 @@ static int32_t tqInitialize(STQ* pTq); static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_EXEC;} static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_IDLE;} -static FORCE_INLINE void tqSetHandleDelete(STqHandle* pHandle) {pHandle->status = TMQ_HANDLE_STATUS_DELETE;} int32_t tqInit() { int8_t old; From cf2c5f190c865003cc7c8132a158840c1e07d8b9 Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Wed, 17 May 2023 19:26:34 +0800 Subject: [PATCH 019/106] Update cases.task modify non asan mode --- tests/parallel_test/cases.task | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 5a6ec7a825..8d151f4756 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -78,7 +78,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +,,n,system-test,python3 ./test.py -f 7-tmq/tmqConsumerGroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAlterSchema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 From 4d5bd2b6dee5321d3aa4695c5553f1866cf4a37e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 May 2023 19:40:21 +0800 Subject: [PATCH 020/106] fix(stream): fix memory leak for stream processing. --- source/libs/stream/src/stream.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 7047461ca8..f4ee0b98b7 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -297,6 +297,7 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); streamDataSubmitDestroy(pSubmitBlock); + taosFreeQitem(pSubmitBlock); return -1; } From 54f48601394cf1dff9f8933d57a3887aaa8ec0d4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 May 2023 19:44:56 +0800 Subject: [PATCH 021/106] fix:rollback [TD-23972] push subscribe msg to vnode even though consumer not change --- source/dnode/mnode/impl/src/mndSubscribe.c | 21 +++------------ source/dnode/vnode/src/tq/tq.c | 30 ++++++++++------------ 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e62102fa77..2ae4ede25a 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -133,10 +133,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg) { -// if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { -// terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; -// return -1; -// } + if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { + terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; + return -1; + } void *buf; int32_t tlen; @@ -269,18 +269,6 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { } } -static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ - for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ - SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); - SMqRebOutputVg outputVg = { - .oldConsumerId = pConsumerEp->consumerId, - .newConsumerId = pConsumerEp->consumerId, - .pVgEp = pVgEp, - }; - taosArrayPush(pOutput->rebVgs, &outputVg); - } -} - static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, int32_t imbConsumerNum) { const char *pSubKey = pOutput->pSub->key; @@ -330,7 +318,6 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas } } } - putNoTransferToOutput(pOutput, pConsumerEp); } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d8ebf72549..e074a2d8a6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -564,29 +564,27 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg goto end; } else { if (pHandle->consumerId == req.newConsumerId) { // do nothing - tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); + tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs, should not reach here", req.vgId, req.newConsumerId); // atomic_add_fetch_32(&pHandle->epoch, 1); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); // atomic_store_32(&pHandle->epoch, 0); + // kill executing task + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + } + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } + // remove if it has been register in the push manager, and return one empty block to consumer + taosWLockLatch(&pTq->lock); + tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } - // remove if it has been register in the push manager, and return one empty block to consumer - taosWLockLatch(&pTq->lock); - tqUnregisterPushHandle(pTq, pHandle); - taosWUnLockLatch(&pTq->lock); - - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); - goto end; } end: From 8994081592b788630e784fa3d7894ba9b9cffeff Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 May 2023 10:25:10 +0800 Subject: [PATCH 022/106] fix: memory size overflow issue --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fa46715c22..5953074825 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -135,7 +135,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in size_t keyLen = 0; int32_t iter = 0; - int32_t bufLen = 0, offset = 0; + int64_t bufLen = 0, offset = 0; // todo move away and record this during create window while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { From ee16b46112e6a83eb1434b9ab90cc104c8e89bcb Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 10:46:19 +0800 Subject: [PATCH 023/106] fix: remove group keys from join operator --- source/libs/executor/src/joinoperator.c | 94 +++++-------------------- 1 file changed, 18 insertions(+), 76 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 07b8a4db11..68f6951c26 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -55,12 +55,10 @@ typedef struct SJoinOperatorInfo { SNode* pTagEqualConditions; SArray* leftTagCols; - SArray* leftTagKeys; char* leftTagKeyBuf; int32_t leftTagKeyLen; SArray* rightTagCols; - SArray* rightTagKeys; char* rightTagKeyBuf; int32_t rightTagKeyLen; @@ -141,27 +139,11 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD } } -static int32_t initTagColsGroupkeys(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { - *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); - if ((*pGroupColVals) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - +static int32_t initTagColskeyBuf(int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); (*keyLen) += pCol->bytes; // actual data + null_flag - - SGroupKeys key = {0}; - key.bytes = pCol->bytes; - key.type = pCol->type; - key.isNull = false; - key.pData = taosMemoryCalloc(1, pCol->bytes); - if (key.pData == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - taosArrayPush((*pGroupColVals), &key); } int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; @@ -175,10 +157,11 @@ static int32_t initTagColsGroupkeys(SArray** pGroupColVals, int32_t* keyLen, cha return TSDB_CODE_SUCCESS; } -static void fillGroupKeyValsFromTagCols(SArray* pCols, SArray* pGroupKeys, SSDataBlock* pBlock, int32_t rowIndex) { +static int32_t fillKeyBufFromTagCols(SArray* pCols, SSDataBlock* pBlock, int32_t rowIndex, void* pKey) { SColumnDataAgg* pColAgg = NULL; - size_t numOfGroupCols = taosArrayGetSize(pCols); + char* isNull = (char*)pKey; + char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = (SColumn*) taosArrayGet(pCols, i); @@ -193,56 +176,24 @@ static void fillGroupKeyValsFromTagCols(SArray* pCols, SArray* pGroupKeys, SSDat pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } - SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { - pkey->isNull = true; + isNull[i] = 1; } else { - pkey->isNull = false; + isNull[i] = 0; char* val = colDataGetData(pColInfoData, rowIndex); - if (pkey->type == TSDB_DATA_TYPE_JSON) { - if (tTagIsJson(val)) { - terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; - return; - } + if (pCol->type == TSDB_DATA_TYPE_JSON) { int32_t dataLen = getJsonValueLen(val); - memcpy(pkey->pData, val, dataLen); - } else if (IS_VAR_DATA_TYPE(pkey->type)) { - memcpy(pkey->pData, val, varDataTLen(val)); - ASSERT(varDataTLen(val) <= pkey->bytes); + memcpy(pStart, val, dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + varDataCopy(pStart, val); + pStart += varDataTLen(val); } else { - memcpy(pkey->pData, val, pkey->bytes); + memcpy(pStart, val, pCol->bytes); + pStart += pCol->bytes; } } } -} - -static int32_t combineGroupKeysIntoBuf(void* pKey, const SArray* pGroupKeys) { - size_t numOfGroupCols = taosArrayGetSize(pGroupKeys); - - char* isNull = (char*)pKey; - char* pStart = (char*)pKey + sizeof(int8_t) * numOfGroupCols; - for (int32_t i = 0; i < numOfGroupCols; ++i) { - SGroupKeys* pkey = taosArrayGet(pGroupKeys, i); - if (pkey->isNull) { - isNull[i] = 1; - continue; - } - - isNull[i] = 0; - if (pkey->type == TSDB_DATA_TYPE_JSON) { - int32_t dataLen = getJsonValueLen(pkey->pData); - memcpy(pStart, (pkey->pData), dataLen); - pStart += dataLen; - } else if (IS_VAR_DATA_TYPE(pkey->type)) { - varDataCopy(pStart, pkey->pData); - pStart += varDataTLen(pkey->pData); - ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); - } else { - memcpy(pStart, pkey->pData, pkey->bytes); - pStart += pkey->bytes; - } - } - return (int32_t)(pStart - (char*)pKey); } @@ -312,8 +263,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); - initTagColsGroupkeys(&pInfo->leftTagKeys, &pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); - initTagColsGroupkeys(&pInfo->rightTagKeys, &pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + initTagColskeyBuf(&pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); + initTagColskeyBuf(&pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); @@ -341,20 +292,13 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } -static void freeGroupKeyData(void* param) { - SGroupKeys* pKey = (SGroupKeys*)param; - taosMemoryFree(pKey->pData); -} - void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; if (pJoinOperator->pTagEqualConditions != NULL) { taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->rightTagKeys, freeGroupKeyData); taosArrayDestroy(pJoinOperator->rightTagCols); taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); - taosArrayDestroyEx(pJoinOperator->leftTagKeys, freeGroupKeyData); taosArrayDestroy(pJoinOperator->leftTagCols); } nodesDestroyNode(pJoinOperator->pCondAfterMerge); @@ -482,8 +426,7 @@ static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* right SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); - fillGroupKeyValsFromTagCols(pInfo->rightTagCols, pInfo->rightTagKeys, rightRow->pDataBlock, rightRow->pos); - int32_t keyLen = combineGroupKeysIntoBuf(pInfo->rightTagKeyBuf, pInfo->rightTagKeys); + int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); SArray** ppRows = tSimpleHashGet(buildTable, pInfo->rightTagKeyBuf, keyLen); if (!ppRows) { SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); @@ -511,8 +454,7 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SArray* pRightRows = NULL; if (rightTableHash != NULL) { - fillGroupKeyValsFromTagCols(pJoinInfo->leftTagCols, pJoinInfo->leftTagKeys, leftRow->pDataBlock, leftRow->pos); - int32_t keyLen = combineGroupKeysIntoBuf(pJoinInfo->leftTagKeyBuf, pJoinInfo->leftTagKeys); + int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftTagCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftTagKeyBuf); SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); if (!ppRightRows) { continue; From 14d4a81e750303da755b121da8c2c36dfac93dff Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 11:15:04 +0800 Subject: [PATCH 024/106] fix: build hash table is created during creating join operator --- source/libs/executor/src/joinoperator.c | 62 +++++++++++++------------ 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 68f6951c26..c90c18e6f5 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -35,7 +35,7 @@ typedef struct SJoinRowCtx { int32_t leftRowIdx; int32_t rightRowIdx; - SSHashObj* buildTableTSRange; + bool rightUseBuildTable; SArray* rightRowLocations; } SJoinRowCtx; @@ -62,6 +62,7 @@ typedef struct SJoinOperatorInfo { char* rightTagKeyBuf; int32_t rightTagKeyLen; + SSHashObj* rightBuildTable; SJoinRowCtx rowCtx; } SJoinOperatorInfo; @@ -265,6 +266,8 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); initTagColskeyBuf(&pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); initTagColskeyBuf(&pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->rightBuildTable = tSimpleHashInit(256, hashFn); } pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); @@ -292,9 +295,22 @@ void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode) { pColumn->scale = pColumnNode->node.resType.scale; } +static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { + void* p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } + + tSimpleHashCleanup(pBuildTable); +} + void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; if (pJoinOperator->pTagEqualConditions != NULL) { + mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); taosArrayDestroy(pJoinOperator->rightTagCols); @@ -420,10 +436,7 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } -static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj** ppHashObj) { - int32_t buildTableCap = TMIN(taosArrayGetSize(rightRowLocations), 256); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* buildTable = tSimpleHashInit(buildTableCap, hashFn); +static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj* buildTable) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); @@ -436,13 +449,12 @@ static int32_t mergeJoinCreateBuildTable(SJoinOperatorInfo* pInfo, SArray* right taosArrayPush(*ppRows, rightRow); } } - *ppHashObj = buildTable; return TSDB_CODE_SUCCESS; } static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* pRes, int32_t* nRows, const SArray* leftRowLocations, int32_t leftRowIdx, - int32_t rightRowIdx, SSHashObj* rightTableHash, SArray* rightRowLocations, bool* pReachThreshold) { + int32_t rightRowIdx, bool useBuildTableTSRange, SArray* rightRowLocations, bool* pReachThreshold) { *pReachThreshold = false; uint32_t limitRowNum = pOperator->resultInfo.threshold; SJoinOperatorInfo* pJoinInfo = pOperator->info; @@ -453,9 +465,9 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* for (i = leftRowIdx; i < leftNumJoin; ++i, rightRowIdx = 0) { SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SArray* pRightRows = NULL; - if (rightTableHash != NULL) { + if (useBuildTableTSRange) { int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftTagCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftTagKeyBuf); - SArray** ppRightRows = tSimpleHashGet(rightTableHash, pJoinInfo->leftTagKeyBuf, keyLen); + SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftTagKeyBuf, keyLen); if (!ppRightRows) { continue; } @@ -488,20 +500,8 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* return TSDB_CODE_SUCCESS; } -static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { - void* p = NULL; - int32_t iter = 0; - - while ((p = tSimpleHashIterate(pBuildTable, p, &iter)) != NULL) { - SArray* rows = (*(SArray**)p); - taosArrayDestroy(rows); - } - - tSimpleHashCleanup(pBuildTable); -} - static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* leftRowLocations, SArray* leftCreatedBlocks, - SArray* rightCreatedBlocks, SSHashObj* rightTableHash, SArray* rightRowLocations) { + SArray* rightCreatedBlocks, bool rightUseBuildTable, SArray* rightRowLocations) { for (int i = 0; i < taosArrayGetSize(rightCreatedBlocks); ++i) { SSDataBlock* pBlock = taosArrayGetP(rightCreatedBlocks, i); blockDataDestroy(pBlock); @@ -514,8 +514,8 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef if (rightRowLocations != NULL) { taosArrayDestroy(rightRowLocations); } - if (rightTableHash != NULL) { - mergeJoinDestoryBuildTable(rightTableHash); + if (rightUseBuildTable) { + tSimpleHashClear(pJoinInfo->rightBuildTable); } taosArrayDestroy(leftCreatedBlocks); @@ -525,7 +525,7 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef pJoinInfo->rowCtx.leftRowLocations = NULL; pJoinInfo->rowCtx.leftCreatedBlocks = NULL; pJoinInfo->rowCtx.rightCreatedBlocks = NULL; - pJoinInfo->rowCtx.buildTableTSRange = NULL; + pJoinInfo->rowCtx.rightUseBuildTable = false; pJoinInfo->rowCtx.rightRowLocations = NULL; } @@ -540,11 +540,12 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t int32_t leftRowIdx = 0; int32_t rightRowIdx = 0; SSHashObj* rightTableHash = NULL; + bool rightUseBuildTable = false; if (pJoinInfo->rowCtx.rowRemains) { leftRowLocations = pJoinInfo->rowCtx.leftRowLocations; leftCreatedBlocks = pJoinInfo->rowCtx.leftCreatedBlocks; - rightTableHash = pJoinInfo->rowCtx.buildTableTSRange; + rightUseBuildTable = pJoinInfo->rowCtx.rightUseBuildTable; rightRowLocations = pJoinInfo->rowCtx.rightRowLocations; rightCreatedBlocks = pJoinInfo->rowCtx.rightCreatedBlocks; leftRowIdx = pJoinInfo->rowCtx.leftRowIdx; @@ -561,7 +562,8 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { - mergeJoinCreateBuildTable(pJoinInfo, rightRowLocations, &rightTableHash); + mergeJoinFillBuildTable(pJoinInfo, rightRowLocations, pJoinInfo->rightBuildTable); + rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); rightRowLocations = NULL; } @@ -578,12 +580,12 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t if (code == TSDB_CODE_SUCCESS) { mergeJoinLeftRowsRightRows(pOperator, pRes, nRows, leftRowLocations, leftRowIdx, - rightRowIdx, rightTableHash, rightRowLocations, &reachThreshold); + rightRowIdx, rightUseBuildTable, rightRowLocations, &reachThreshold); } if (!reachThreshold) { mergeJoinDestroyTSRangeCtx(pJoinInfo, leftRowLocations, leftCreatedBlocks, rightCreatedBlocks, - rightTableHash, rightRowLocations); + rightUseBuildTable, rightRowLocations); } else { pJoinInfo->rowCtx.rowRemains = true; @@ -591,7 +593,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - pJoinInfo->rowCtx.buildTableTSRange = rightTableHash; + pJoinInfo->rowCtx.rightUseBuildTable = true; pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; From d7f8757844da2a8db1d9a756d2d9884a3b57cb91 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 11:23:50 +0800 Subject: [PATCH 025/106] fix: memory leak --- source/libs/executor/src/joinoperator.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index c90c18e6f5..0add034673 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -515,6 +515,12 @@ static void mergeJoinDestroyTSRangeCtx(SJoinOperatorInfo* pJoinInfo, SArray* lef taosArrayDestroy(rightRowLocations); } if (rightUseBuildTable) { + void* p = NULL; + int32_t iter = 0; + while ((p = tSimpleHashIterate(pJoinInfo->rightBuildTable, p, &iter)) != NULL) { + SArray* rows = (*(SArray**)p); + taosArrayDestroy(rows); + } tSimpleHashClear(pJoinInfo->rightBuildTable); } From 7102c107ebd4884541bfa47d428b73020b5a7d43 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 13:19:52 +0800 Subject: [PATCH 026/106] fix: remove parameter of filling build table --- source/libs/executor/src/joinoperator.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 0add034673..23bd7c5700 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -436,15 +436,16 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator return 0; } -static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations,SSHashObj* buildTable) { +static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { + tSimpleHashClear(pInfo->rightBuildTable); SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); - SArray** ppRows = tSimpleHashGet(buildTable, pInfo->rightTagKeyBuf, keyLen); + SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen); if (!ppRows) { SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); taosArrayPush(rows, rightRow); - tSimpleHashPut(buildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); + tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); } else { taosArrayPush(*ppRows, rightRow); } @@ -568,7 +569,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { - mergeJoinFillBuildTable(pJoinInfo, rightRowLocations, pJoinInfo->rightBuildTable); + mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); rightRowLocations = NULL; From 51cb79fc1d93eec01fba74a513c7e7993c3c8e98 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 13:28:46 +0800 Subject: [PATCH 027/106] fix: set rightUseBuildTable correctly --- source/libs/executor/src/joinoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 23bd7c5700..700a59ccd8 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -600,7 +600,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->rowCtx.leftRowLocations = leftRowLocations; pJoinInfo->rowCtx.leftCreatedBlocks = leftCreatedBlocks; pJoinInfo->rowCtx.rightCreatedBlocks = rightCreatedBlocks; - pJoinInfo->rowCtx.rightUseBuildTable = true; + pJoinInfo->rowCtx.rightUseBuildTable = rightUseBuildTable; pJoinInfo->rowCtx.rightRowLocations = rightRowLocations; } return TSDB_CODE_SUCCESS; From 72e63195920db702e3b1165949d1f13e56d84c53 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 May 2023 14:41:32 +0800 Subject: [PATCH 028/106] enh: optimize order by for partition by clause --- source/libs/planner/src/planLogicCreater.c | 23 ++++--- tests/script/tsim/query/partitionby.sim | 79 ++++++++++++++++++++-- 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b9ea5ba0cd..91a5e6ac93 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -37,19 +37,24 @@ typedef struct SRewriteExprCxt { int32_t errCode; SNodeList* pExprs; bool* pOutputs; + bool isPartitionBy; } SRewriteExprCxt; -static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol) { +static void setColumnInfo(SFunctionNode* pFunc, SColumnNode* pCol, bool isPartitionBy) { switch (pFunc->funcType) { case FUNCTION_TYPE_TBNAME: pCol->colType = COLUMN_TYPE_TBNAME; break; case FUNCTION_TYPE_WSTART: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_START; break; case FUNCTION_TYPE_WEND: - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (!isPartitionBy) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } pCol->colType = COLUMN_TYPE_WINDOW_END; break; case FUNCTION_TYPE_WDURATION: @@ -103,7 +108,7 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { strcpy(pCol->node.userAlias, ((SExprNode*)pExpr)->userAlias); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { - setColumnInfo((SFunctionNode*)pExpr, pCol); + setColumnInfo((SFunctionNode*)pExpr, pCol, pCxt->isPartitionBy); } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; @@ -142,7 +147,8 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { static int32_t rewriteExprForSelect(SNode* pExpr, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExpr(pExpr, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = NULL, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; cxt.errCode = nodesListMakeAppend(&cxt.pExprs, pExpr); if (TSDB_CODE_SUCCESS == cxt.errCode) { nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); @@ -170,7 +176,8 @@ static int32_t cloneRewriteExprs(SNodeList* pExprs, bool* pOutputs, SNodeList** static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause, SNodeList** pRewriteExprs) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + bool isPartitionBy = (pSelect->pPartitionByList && pSelect->pPartitionByList->length > 0) ? true : false; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = isPartitionBy}; if (NULL != pRewriteExprs) { cxt.pOutputs = taosMemoryCalloc(LIST_LENGTH(pExprs), sizeof(bool)); if (NULL == cxt.pOutputs) { @@ -187,14 +194,14 @@ static int32_t rewriteExprsForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ES static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { nodesWalkExprs(pExprs, doNameExpr, NULL); - SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL}; + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs, .pOutputs = NULL, .isPartitionBy = false}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; } diff --git a/tests/script/tsim/query/partitionby.sim b/tests/script/tsim/query/partitionby.sim index 8babd1aa8d..4657713780 100644 --- a/tests/script/tsim/query/partitionby.sim +++ b/tests/script/tsim/query/partitionby.sim @@ -24,16 +24,83 @@ sql use $db sql create table $mt1 (ts timestamp, f1 int) TAGS(tag1 int, tag2 binary(500)) sql create table tb0 using $mt1 tags(0, 'a'); sql create table tb1 using $mt1 tags(1, 'b'); -sql create table tb2 using $mt1 tags(1, 'a'); -sql create table tb3 using $mt1 tags(1, 'a'); -sql create table tb4 using $mt1 tags(3, 'b'); -sql create table tb5 using $mt1 tags(3, 'a'); -sql create table tb6 using $mt1 tags(3, 'b'); -sql create table tb7 using $mt1 tags(3, 'b'); +sql create table tb2 using $mt1 tags(2, 'a'); +sql create table tb3 using $mt1 tags(3, 'a'); +sql create table tb4 using $mt1 tags(4, 'b'); +sql create table tb5 using $mt1 tags(5, 'a'); +sql create table tb6 using $mt1 tags(6, 'b'); +sql create table tb7 using $mt1 tags(7, 'b'); sql select * from $mt1 partition by tag1,tag2 limit 1; if $rows != 0 then return -1 endi +sql insert into tb0 values ('2022-04-26 15:15:08', 1); +sql insert into tb1 values ('2022-04-26 15:15:07', 2); +sql insert into tb2 values ('2022-04-26 15:15:06', 3); +sql insert into tb3 values ('2022-04-26 15:15:05', 4); +sql insert into tb4 values ('2022-04-26 15:15:04', 5); +sql insert into tb5 values ('2022-04-26 15:15:03', 6); +sql insert into tb6 values ('2022-04-26 15:15:02', 7); +sql insert into tb7 values ('2022-04-26 15:15:01', 8); + +sql select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart; +if $rows != 8 then + return -1 +endi +if $data00 != @22-04-26 15:15:01.000@ then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data10 != @22-04-26 15:15:02.000@ then + return -1 +endi +if $data11 != 1 then + return -1 +endi +if $data20 != @22-04-26 15:15:03.000@ then + return -1 +endi +if $data21 != 1 then + return -1 +endi +if $data30 != @22-04-26 15:15:04.000@ then + return -1 +endi +if $data31 != 1 then + return -1 +endi +if $data40 != @22-04-26 15:15:05.000@ then + return -1 +endi +if $data41 != 1 then + return -1 +endi +if $data50 != @22-04-26 15:15:06.000@ then + return -1 +endi +if $data51 != 1 then + return -1 +endi +if $data60 != @22-04-26 15:15:07.000@ then + return -1 +endi +if $data61 != 1 then + return -1 +endi +if $data70 != @22-04-26 15:15:08.000@ then + return -1 +endi +if $data71 != 1 then + return -1 +endi +sql select * from (select _wstart as ts, count(*) from $mt1 partition by tag1 interval(1s) order by _wstart) order by ts; +sql select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart; +sql select * from (select _wstart as ts, count(*) from $mt1 interval(1s) order by _wstart) order by ts; +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 interval(1s) order by _wstart); +sql select diff(a) from (select _wstart as ts, count(*) a from $mt1 partition by tag1 interval(1s) order by _wstart); + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 4db88e8ffe49e5dbbb56caf25b51d157dc13042e Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 15:45:58 +0800 Subject: [PATCH 029/106] fix: fix ci comments --- source/libs/executor/src/joinoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 700a59ccd8..9dee58367c 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -438,7 +438,6 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { - tSimpleHashClear(pInfo->rightBuildTable); SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen); From 99b424b181d527a3ce368b199dcf8be0a142d70d Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 16:26:20 +0800 Subject: [PATCH 030/106] fix: add test case --- tests/script/output.txt | 6 ++ tests/script/sh/l2norm2.c | 71 ++++++++++++++++++++ tests/script/tsim/parser/join_manyblocks.sim | 8 +-- 3 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 tests/script/output.txt create mode 100644 tests/script/sh/l2norm2.c diff --git a/tests/script/output.txt b/tests/script/output.txt new file mode 100644 index 0000000000..0c7c386bbc --- /dev/null +++ b/tests/script/output.txt @@ -0,0 +1,6 @@ +[03/30 08:08:52.140115] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection +[03/30 08:08:52.140136] ERROR: insert test process failed +[03/30 08:09:02.860424] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection +[03/30 08:09:02.860445] ERROR: insert test process failed +[03/30 08:10:57.082603] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection +[03/30 08:10:57.082626] ERROR: insert test process failed diff --git a/tests/script/sh/l2norm2.c b/tests/script/sh/l2norm2.c new file mode 100644 index 0000000000..0739f3a23b --- /dev/null +++ b/tests/script/sh/l2norm2.c @@ -0,0 +1,71 @@ +#include +#include +#include +#include + +#include "taosudf.h" + +DLL_EXPORT int32_t l2norm2_init() { + return 0; +} + +DLL_EXPORT int32_t l2norm2_destroy() { + return 0; +} + +DLL_EXPORT int32_t l2norm2_start(SUdfInterBuf *buf) { + *(int64_t*)(buf->buf) = 0; + buf->bufLen = sizeof(double); + buf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t l2norm2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { + double sumSquares = *(double*)interBuf->buf; + int8_t numNotNull = 0; + for (int32_t i = 0; i < block->numOfCols; ++i) { + SUdfColumn* col = block->udfCols[i]; + if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || + col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } + for (int32_t i = 0; i < block->numOfCols; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { + SUdfColumn* col = block->udfCols[i]; + if (udfColDataIsNull(col, j)) { + continue; + } + switch (col->colMeta.type) { + case TSDB_DATA_TYPE_INT: { + char* cell = udfColDataGetData(col, j); + int32_t num = *(int32_t*)cell; + sumSquares += (double)num * num; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + char* cell = udfColDataGetData(col, j); + double num = *(double*)cell; + sumSquares += num * num; + break; + } + default: + break; + } + ++numNotNull; + } + } + + *(double*)(newInterBuf->buf) = sumSquares; + newInterBuf->bufLen = sizeof(double); + newInterBuf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t l2norm2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { + double sumSquares = *(double*)(buf->buf); + *(double*)(resultData->buf) = sumSquares; + resultData->bufLen = sizeof(double); + resultData->numOfResult = 1; + return 0; +} diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index a40a75f50c..7fd0df21b3 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -6,8 +6,8 @@ sql connect $dbPrefix = join_m_db $tbPrefix = join_tb $mtPrefix = join_mt -$tbNum = 3 -$rowNum = 2000 +$tbNum = 20 +$rowNum = 200 $totalNum = $tbNum * $rowNum print =============== join_manyBlocks.sim @@ -78,8 +78,8 @@ print ==============> td-3313 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; print $row -if $row != 6000 then - print expect 6000, actual: $row +if $row != 4000 then + print expect 4000, actual: $row return -1 endi From 3e02bb8faa96dfb7c23fe34357d47231e1033dd9 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 May 2023 16:26:52 +0800 Subject: [PATCH 031/106] fix: remove files --- tests/script/output.txt | 6 ---- tests/script/sh/l2norm2.c | 71 --------------------------------------- 2 files changed, 77 deletions(-) delete mode 100644 tests/script/output.txt delete mode 100644 tests/script/sh/l2norm2.c diff --git a/tests/script/output.txt b/tests/script/output.txt deleted file mode 100644 index 0c7c386bbc..0000000000 --- a/tests/script/output.txt +++ /dev/null @@ -1,6 +0,0 @@ -[03/30 08:08:52.140115] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection -[03/30 08:08:52.140136] ERROR: insert test process failed -[03/30 08:09:02.860424] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection -[03/30 08:09:02.860445] ERROR: insert test process failed -[03/30 08:10:57.082603] ERROR: failed to connect native (null):6030, code: 0x8000000b, reason: Unable to establish connection -[03/30 08:10:57.082626] ERROR: insert test process failed diff --git a/tests/script/sh/l2norm2.c b/tests/script/sh/l2norm2.c deleted file mode 100644 index 0739f3a23b..0000000000 --- a/tests/script/sh/l2norm2.c +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include -#include - -#include "taosudf.h" - -DLL_EXPORT int32_t l2norm2_init() { - return 0; -} - -DLL_EXPORT int32_t l2norm2_destroy() { - return 0; -} - -DLL_EXPORT int32_t l2norm2_start(SUdfInterBuf *buf) { - *(int64_t*)(buf->buf) = 0; - buf->bufLen = sizeof(double); - buf->numOfResult = 1; - return 0; -} - -DLL_EXPORT int32_t l2norm2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { - double sumSquares = *(double*)interBuf->buf; - int8_t numNotNull = 0; - for (int32_t i = 0; i < block->numOfCols; ++i) { - SUdfColumn* col = block->udfCols[i]; - if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || - col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) { - return TSDB_CODE_UDF_INVALID_INPUT; - } - } - for (int32_t i = 0; i < block->numOfCols; ++i) { - for (int32_t j = 0; j < block->numOfRows; ++j) { - SUdfColumn* col = block->udfCols[i]; - if (udfColDataIsNull(col, j)) { - continue; - } - switch (col->colMeta.type) { - case TSDB_DATA_TYPE_INT: { - char* cell = udfColDataGetData(col, j); - int32_t num = *(int32_t*)cell; - sumSquares += (double)num * num; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - char* cell = udfColDataGetData(col, j); - double num = *(double*)cell; - sumSquares += num * num; - break; - } - default: - break; - } - ++numNotNull; - } - } - - *(double*)(newInterBuf->buf) = sumSquares; - newInterBuf->bufLen = sizeof(double); - newInterBuf->numOfResult = 1; - return 0; -} - -DLL_EXPORT int32_t l2norm2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { - double sumSquares = *(double*)(buf->buf); - *(double*)(resultData->buf) = sumSquares; - resultData->bufLen = sizeof(double); - resultData->numOfResult = 1; - return 0; -} From 79623860a43cdd9604e7e41037bdb5db6044d393 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 18 May 2023 17:41:01 +0800 Subject: [PATCH 032/106] fix: distinct wrong result issue --- source/libs/planner/src/planLogicCreater.c | 9 ++++-- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/groupby_distinct.sim | 30 ++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 tests/script/tsim/query/groupby_distinct.sim diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index b9ea5ba0cd..16bd16238e 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -544,11 +544,16 @@ static SNode* createGroupingSetNode(SNode* pExpr) { return (SNode*)pGroupingSet; } -static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { +static EGroupAction getDistinctGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { return (pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) ? GROUP_ACTION_KEEP : GROUP_ACTION_NONE; } +static EGroupAction getGroupAction(SLogicPlanContext* pCxt, SSelectStmt* pSelect) { + return ((pCxt->pPlanCxt->streamQuery || NULL != pSelect->pLimit || NULL != pSelect->pSlimit) && !pSelect->isDistinct) ? GROUP_ACTION_KEEP + : GROUP_ACTION_NONE; +} + static EDataOrderLevel getRequireDataOrder(bool needTimeline, SSelectStmt* pSelect) { return needTimeline ? (NULL != pSelect->pPartitionByList ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL) : DATA_ORDER_LEVEL_NONE; @@ -1152,7 +1157,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = GROUP_ACTION_CLEAR; + pAgg->node.groupAction = getDistinctGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e79cf77648..bcfb392bb4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -900,6 +900,7 @@ ,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim +,,y,script,./test.sh -f tsim/query/groupby_distinct.sim ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim diff --git a/tests/script/tsim/query/groupby_distinct.sim b/tests/script/tsim/query/groupby_distinct.sim new file mode 100644 index 0000000000..8b16bb1910 --- /dev/null +++ b/tests/script/tsim/query/groupby_distinct.sim @@ -0,0 +1,30 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + + +sql drop database if exists db1; +sql create database db1; +sql use db1; + +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql insert into tba1 values ('2022-04-26 15:15:08', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:07', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:06', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:05', 1, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:03', 1, "c"); +sql insert into tba1 values ('2022-04-26 15:15:02', 1, "d"); +sql insert into tba1 values ('2022-04-26 15:15:01', 1, "d"); +sql select distinct avg(f1) as avgv from sta group by f2; +if $rows != 1 then + return -1 +endi +sql select distinct avg(f1) as avgv from sta group by f2 limit 1,10; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From c7e6883ac8e2f47a2514246abecd675781f88f91 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 18:10:33 +0800 Subject: [PATCH 033/106] fix:tmq ref err & vg error --- source/client/src/clientTmq.c | 101 +++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 20 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index b488af9ba1..be547c42b2 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -87,6 +87,7 @@ struct tmq_t { void* commitCbUserParam; // status + SRWLatch lock; int8_t status; int32_t epoch; #if 0 @@ -156,6 +157,7 @@ typedef struct { int8_t tmqRspType; int32_t epoch; // epoch can be used to guard the vgHandle int32_t vgId; + char topicName[TSDB_TOPIC_FNAME_LEN]; SMqClientVg* vgHandle; SMqClientTopic* topicHandle; uint64_t reqId; @@ -168,8 +170,8 @@ typedef struct { } SMqPollRspWrapper; typedef struct { - int64_t refId; - int32_t epoch; +// int64_t refId; +// int32_t epoch; tsem_t rspSem; int32_t rspErr; } SMqSubscribeCbParam; @@ -184,8 +186,9 @@ typedef struct { typedef struct { int64_t refId; int32_t epoch; - SMqClientVg* pVg; - SMqClientTopic* pTopic; + char topicName[TSDB_TOPIC_FNAME_LEN]; +// SMqClientVg* pVg; +// SMqClientTopic* pTopic; int32_t vgId; uint64_t requestId; // request id for debug purpose } SMqPollCbParam; @@ -709,8 +712,8 @@ static void generateTimedTask(int64_t refId, int32_t type) { *pTaskType = type; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); } - taosReleaseRef(tmqMgmt.rsetId, refId); } void tmqAssignAskEpTask(void* param, void* tmrId) { @@ -1037,6 +1040,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->commitCb = conf->commitCb; pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; + taosInitRWLatch(&pTmq->lock); pTmq->hbBgEnable = conf->hbBgEnable; @@ -1140,8 +1144,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SMqSubscribeCbParam param = { .rspErr = 0, - .refId = tmq->refId, - .epoch = tmq->epoch, +// .refId = tmq->refId, +// .epoch = tmq->epoch, }; if (tsem_init(¶m.rspSem, 0, 0) != 0) { @@ -1217,12 +1221,40 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para conf->commitCbUserParam = param; } +static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + for (int32_t j = 0; j < vgNumCur; j++) { + SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); + if(pVgCur->vgId == vgId){ + return pVgCur; + } + } + } + } + return NULL; +} + +static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName){ + int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); + for(int i = 0; i < topicNumCur; i++){ + SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); + if(strcmp(pTopicCur->topicName, topicName) == 0){ + return pTopicCur; + } + } + return NULL; +} + int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; int64_t refId = pParam->refId; - SMqClientVg* pVg = pParam->pVg; - SMqClientTopic* pTopic = pParam->pTopic; +// SMqClientVg* pVg = pParam->pVg; +// SMqClientTopic* pTopic = pParam->pTopic; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { @@ -1303,11 +1335,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } pRspWrapper->tmqRspType = rspType; - pRspWrapper->vgHandle = pVg; - pRspWrapper->topicHandle = pTopic; +// pRspWrapper->vgHandle = pVg; +// pRspWrapper->topicHandle = pTopic; pRspWrapper->reqId = requestId; pRspWrapper->pEpset = pMsg->pEpSet; - pRspWrapper->vgId = pVg->vgId; + pRspWrapper->vgId = vgId; + strcpy(pRspWrapper->topicName, pParam->topicName); pMsg->pEpSet = NULL; if (rspType == TMQ_MSG_TYPE__POLL_RSP) { @@ -1351,7 +1384,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { CREATE_MSG_FAIL: if (epoch == tmq->epoch) { - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = getVgInfo(tmq, pParam->topicName, vgId); + if(pVg){ + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + } + taosWLockLatch(&tmq->lock); } tsem_post(&tmq->rspSem); @@ -1472,11 +1510,13 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) taosHashCleanup(pVgOffsetHashMap); + taosWLockLatch(&tmq->lock); // destroy current buffered existed topics info if (tmq->clientTopics) { taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo); } tmq->clientTopics = newTopics; + taosWUnLockLatch(&tmq->lock); int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; atomic_store_8(&tmq->status, flag); @@ -1492,7 +1532,7 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { if (tmq == NULL) { terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); +// pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -1652,8 +1692,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam->refId = pTmq->refId; pParam->epoch = pTmq->epoch; - pParam->pVg = pVg; // pVg may be released,fix it - pParam->pTopic = pTopic; +// pParam->pVg = pVg; // pVg may be released,fix it +// pParam->pTopic = pTopic; + strcpy(pParam->topicName, pTopic->topicName); pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; @@ -1779,8 +1820,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; if (pDataRsp->head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; - + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } // update the epset if (pollRspWrapper->pEpset != NULL) { SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset); @@ -1829,7 +1876,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } if(pollRspWrapper->metaRsp.rspOffset.type != 0){ // if offset is validate pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; } @@ -1849,7 +1903,14 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { - SMqClientVg* pVg = pollRspWrapper->vgHandle; + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + return NULL; + } if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate pVg->currentOffset = pollRspWrapper->taosxRsp.rspOffset; } From 7db9f1e58ed471bcd22e5d692bd11c47835140d6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 18:21:45 +0800 Subject: [PATCH 034/106] fix:tmq ref err & vg error --- source/client/src/clientTmq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index be547c42b2..433b27d5d9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1269,8 +1269,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t vgId = pParam->vgId; uint64_t requestId = pParam->requestId; - taosMemoryFree(pParam); - if (code != 0) { if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); @@ -1314,6 +1312,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pParam); + return 0; } @@ -1379,6 +1379,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return 0; @@ -1394,6 +1395,7 @@ CREATE_MSG_FAIL: tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pParam); return -1; } From 50bdfef7b6e32134a928fc7e6cd0232b2163054e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 18:34:42 +0800 Subject: [PATCH 035/106] fix:tmq ref err & vg error --- source/client/src/clientTmq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 433b27d5d9..01667f6dc3 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1390,7 +1390,7 @@ CREATE_MSG_FAIL: if(pVg){ atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } - taosWLockLatch(&tmq->lock); + taosWUnLockLatch(&tmq->lock); } tsem_post(&tmq->rspSem); From bca1af3f5d7cd00d98096b1b0f5e052b8e8c0790 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 May 2023 21:18:44 +0800 Subject: [PATCH 036/106] fix:change tmq subscribe try time to 60mins --- source/client/src/clientTmq.c | 4 ++-- source/dnode/vnode/src/tq/tqPush.c | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 01667f6dc3..c08fbd0adf 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1089,7 +1089,7 @@ _failed: } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { - const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most + const int32_t MAX_RETRY_COUNT = 120 * 60; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1186,7 +1186,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { if (retryCnt++ > MAX_RETRY_COUNT) { tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); - code = TSDB_CODE_TSC_INTERNAL_ERROR; + code = TSDB_CODE_MND_CONSUMER_NOT_READY; goto FAIL; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 50d94c5ed5..42c60e0007 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -73,8 +73,11 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) { STqHandle *pHandle = (STqHandle*)handle; int32_t vgId = TD_VID(pTq->pVnode); + if(taosHashGetSize(pTq->pPushMgr) <= 0) { + return 0; + } int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey)); - tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); + tqDebug("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); if(pHandle->msg != NULL) { tqPushDataRsp(pTq, pHandle); From d4b0e102af08e156ca8bf61fa091b38d21ca3933 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 May 2023 08:55:59 +0800 Subject: [PATCH 037/106] fix:do some internal refactor. --- include/common/ttime.h | 2 +- source/common/src/ttime.c | 2 -- source/libs/executor/src/executil.c | 2 +- source/libs/parser/src/parTranslater.c | 8 ++++---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/include/common/ttime.h b/include/common/ttime.h index 69a071d59f..7bbc6a8e56 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -23,7 +23,7 @@ extern "C" { #endif -#define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') +#define IS_CALENDAR_TIME_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') #define TIME_UNIT_NANOSECOND 'b' #define TIME_UNIT_MICROSECOND 'u' diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index de6a18f7c8..165a4ff95b 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -709,8 +709,6 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati return getDuration(*duration, *unit, duration, timePrecision); } -#define IS_CALENDAR_TIME_DURATION(_d) (((_d) == 'n') || ((_d) == 'y')) - int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { if (duration == 0) { return t; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 91f384ce6a..b4c5a90be3 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1759,7 +1759,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); - if (pInterval->slidingUnit != 'n' && pInterval->slidingUnit != 'y') { + if (!IS_CALENDAR_TIME_DURATION(pInterval->slidingUnit)) { tw->skey += pInterval->sliding * factor; tw->ekey = taosTimeAdd(tw->skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; return; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0d052846f7..c61c4fe4d4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3004,7 +3004,7 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* int64_t timeRange = TABS(pFill->timeRange.skey - pFill->timeRange.ekey); int64_t intervalRange = 0; - if (TIME_IS_VAR_DURATION(pInterval->unit)) { + if (IS_CALENDAR_TIME_DURATION(pInterval->unit)) { int64_t f = 1; if (pInterval->unit == 'n') { f = 30LL * MILLISECOND_PER_DAY; @@ -3072,7 +3072,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = TIME_IS_VAR_DURATION(pInter->unit); + bool valInter = IS_CALENDAR_TIME_DURATION(pInter->unit); if (pInter->datum.i <= 0 || (!valInter && pInter->datum.i < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime, getPrecisionStr(precision)); @@ -3086,7 +3086,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->unit == 'n' && pOffset->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter; + bool fixed = !IS_CALENDAR_TIME_DURATION(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { @@ -3098,7 +3098,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* const static int32_t INTERVAL_SLIDING_FACTOR = 100; SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (TIME_IS_VAR_DURATION(pSliding->unit)) { + if (IS_CALENDAR_TIME_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || From 93c4e3fe70a9a989d8b96a457b5240147190b4a7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 May 2023 09:12:16 +0800 Subject: [PATCH 038/106] other: remove empty files. --- source/libs/executor/inc/executorimpl.h | 0 source/libs/executor/src/executorimpl.c | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 source/libs/executor/inc/executorimpl.h delete mode 100644 source/libs/executor/src/executorimpl.c diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c deleted file mode 100644 index e69de29bb2..0000000000 From e882601b67f900dde0790d1e55628dfd7b86e4c5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 May 2023 09:15:48 +0800 Subject: [PATCH 039/106] fix: last table scan issue --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 21b59e376c..4acc784aee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3112,6 +3112,10 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) { + // reset the index in last block when handing a new file + doCleanupTableScanInfo(pScanInfo); + pStatus->mapDataCleaned = true; + bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { return TSDB_CODE_SUCCESS; From 7218a034870adc3f336b4672b88e12639b20daa2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 May 2023 11:16:31 +0800 Subject: [PATCH 040/106] fix: result block ensure capacity when finalize may produce serveral rows --- source/libs/executor/src/executorInt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 1a14d0e78a..303be3d2cd 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -715,7 +715,8 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR pCtx[j].resultInfo->numOfRes = pRow->numOfRows; } } - + + blockDataEnsureCapacity(pBlock, pBlock->info.rows + pCtx[j].resultInfo->numOfRes); int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); if (TAOS_FAILED(code)) { qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); From 6a1eb62dd44956c6cb4a7453c83eb96687e1cefc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 May 2023 11:26:40 +0800 Subject: [PATCH 041/106] fix: distinct group issue --- source/libs/planner/src/planLogicCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 16bd16238e..1ec0abe8df 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1157,7 +1157,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } - pAgg->node.groupAction = getDistinctGroupAction(pCxt, pSelect); + pAgg->node.groupAction = GROUP_ACTION_CLEAR;//getDistinctGroupAction(pCxt, pSelect); pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; From 238254e49ee83723dd274bae5a66d7fe9994d5df Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 14:06:43 +0800 Subject: [PATCH 042/106] fix:add task status check in doQueueScan & checkout consumer in tqProcessPollReq --- source/dnode/vnode/src/tq/tq.c | 22 +++++++++--------- source/libs/executor/src/scanoperator.c | 30 +------------------------ 2 files changed, 11 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e074a2d8a6..41b43026ea 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -358,9 +358,19 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } + // 2. check re-balance status + if (pHandle->consumerId != consumerId) { + tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosWUnLockLatch(&pTq->lock); + return -1; + } + bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; } @@ -370,17 +380,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } - // 2. check re-balance status - if (pHandle->consumerId != consumerId) { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, - consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - code = -1; - goto end; - } - - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); - // 3. update the epoch value int32_t savedEpoch = pHandle->epoch; if (savedEpoch < reqEpoch) { @@ -396,7 +395,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); -end: tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7d3165237e..638662aeb5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1665,37 +1665,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("start to exec queue scan, %s", id); -#if 0 - if (pTaskInfo->streamInfo.submit.msgStr != NULL) { - if (pInfo->tqReader->msg.msgStr == NULL) { - SPackedData submit = pTaskInfo->streamInfo.submit; - if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { - qError("submit msg messed up when initing stream submit block %p", submit.msgStr); - return NULL; - } - } - - blockDataCleanup(pInfo->pRes); - SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - - while (tqNextBlockImpl(pInfo->tqReader)) { - int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL); - if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { - continue; - } - - setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true); - - if (pBlockInfo->rows > 0) { - return pInfo->pRes; - } - } - - pInfo->tqReader->msg = (SPackedData){0}; - pTaskInfo->streamInfo.submit = (SPackedData){0}; + if (isTaskKilled(pTaskInfo)) { return NULL; } -#endif if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); From f36b0be17d0c91dcd45941bd5d753d39293e6856 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 May 2023 14:32:03 +0800 Subject: [PATCH 043/106] fix: join eq conditions not only for tag --- include/libs/nodes/plannodes.h | 16 +++---- source/libs/executor/src/joinoperator.c | 56 +++++++++++----------- source/libs/nodes/src/nodesCloneFuncs.c | 4 +- source/libs/nodes/src/nodesCodeFuncs.c | 12 ++--- source/libs/nodes/src/nodesMsgFuncs.c | 8 ++-- source/libs/nodes/src/nodesUtilFuncs.c | 14 +++--- source/libs/planner/src/planOptimizer.c | 33 ++++++------- source/libs/planner/src/planPhysiCreater.c | 6 +-- 8 files changed, 73 insertions(+), 76 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 0e235191b1..f559ba2b51 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,7 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; - SNode* pTagEqualConditions; + SNode* pEqualOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -406,7 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; - SNode* pTagEqualCondtions; + SNode* pEqualOnCondtions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { @@ -448,7 +448,7 @@ typedef struct SMergePhysiNode { bool groupSort; } SMergePhysiNode; -typedef struct SWinodwPhysiNode { +typedef struct SWindowPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pFuncs; @@ -461,10 +461,10 @@ typedef struct SWinodwPhysiNode { EOrder inputTsOrder; EOrder outputTsOrder; bool mergeDataBlock; -} SWinodwPhysiNode; +} SWindowPhysiNode; typedef struct SIntervalPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t interval; int64_t offset; int64_t sliding; @@ -497,7 +497,7 @@ typedef struct SMultiTableIntervalPhysiNode { } SMultiTableIntervalPhysiNode; typedef struct SSessionWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; int64_t gap; } SSessionWinodwPhysiNode; @@ -506,14 +506,14 @@ typedef SSessionWinodwPhysiNode SStreamSemiSessionWinodwPhysiNode; typedef SSessionWinodwPhysiNode SStreamFinalSessionWinodwPhysiNode; typedef struct SStateWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStateKey; } SStateWinodwPhysiNode; typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; typedef struct SEventWinodwPhysiNode { - SWinodwPhysiNode window; + SWindowPhysiNode window; SNode* pStartCond; SNode* pEndCond; } SEventWinodwPhysiNode; diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 9dee58367c..46c6b24295 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -52,15 +52,15 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; - SNode* pTagEqualConditions; + SNode* pEqualOnConditions; - SArray* leftTagCols; - char* leftTagKeyBuf; - int32_t leftTagKeyLen; + SArray* leftEqOnCondCols; + char* leftEqOnCondKeyBuf; + int32_t leftEqOnCondKeyLen; - SArray* rightTagCols; - char* rightTagKeyBuf; - int32_t rightTagKeyLen; + SArray* rightEqOnCondCols; + char* rightEqOnCondKeyBuf; + int32_t rightEqOnCondKeyLen; SSHashObj* rightBuildTable; SJoinRowCtx rowCtx; @@ -104,7 +104,7 @@ static void extractTimeCondition(SJoinOperatorInfo* pInfo, SOperatorInfo** pDown setJoinColumnInfo(&pInfo->rightCol, rightTsCol); } -static void extractTagEqualColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, +static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownstreams, SOperatorNode* pOperNode, SColumn* pLeft, SColumn* pRight) { SColumnNode* pLeftNode = (SColumnNode*)pOperNode->pLeft; SColumnNode* pRightNode = (SColumnNode*)pOperNode->pRight; @@ -125,7 +125,7 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD SNode* pNode = NULL; FOREACH(pNode, ((SLogicConditionNode*)pTagEqualNode)->pParameterList) { SOperatorNode* pOperNode = (SOperatorNode*)pNode; - extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); taosArrayPush(rightTagEqCols, &right); } @@ -134,7 +134,7 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD if (nodeType(pTagEqualNode) == QUERY_NODE_OPERATOR) { SOperatorNode* pOperNode = (SOperatorNode*)pTagEqualNode; - extractTagEqualColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); + extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); taosArrayPush(rightTagEqCols, &right); } @@ -259,13 +259,13 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pInfo->pTagEqualConditions = pJoinNode->pTagEqualCondtions; - if (pInfo->pTagEqualConditions != NULL) { - pInfo->leftTagCols = taosArrayInit(4, sizeof(SColumn)); - pInfo->rightTagCols = taosArrayInit(4, sizeof(SColumn)); - extractTagEqualCondCols(pInfo, pDownstream, pInfo->pTagEqualConditions, pInfo->leftTagCols, pInfo->rightTagCols); - initTagColskeyBuf(&pInfo->leftTagKeyLen, &pInfo->leftTagKeyBuf, pInfo->leftTagCols); - initTagColskeyBuf(&pInfo->rightTagKeyLen, &pInfo->rightTagKeyBuf, pInfo->rightTagCols); + pInfo->pEqualOnConditions = pJoinNode->pEqualOnCondtions; + if (pInfo->pEqualOnConditions != NULL) { + pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); + extractTagEqualCondCols(pInfo, pDownstream, pInfo->pEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); + initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols); + initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->rightBuildTable = tSimpleHashInit(256, hashFn); } @@ -309,13 +309,13 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; - if (pJoinOperator->pTagEqualConditions != NULL) { + if (pJoinOperator->pEqualOnConditions != NULL) { mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); - taosMemoryFreeClear(pJoinOperator->rightTagKeyBuf); - taosArrayDestroy(pJoinOperator->rightTagCols); + taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->rightEqOnCondCols); - taosMemoryFreeClear(pJoinOperator->leftTagKeyBuf); - taosArrayDestroy(pJoinOperator->leftTagCols); + taosMemoryFreeClear(pJoinOperator->leftEqOnCondKeyBuf); + taosArrayDestroy(pJoinOperator->leftEqOnCondCols); } nodesDestroyNode(pJoinOperator->pCondAfterMerge); @@ -439,12 +439,12 @@ static int32_t mergeJoinGetDownStreamRowsEqualTimeStamp(SOperatorInfo* pOperator static int32_t mergeJoinFillBuildTable(SJoinOperatorInfo* pInfo, SArray* rightRowLocations) { for (int32_t i = 0; i < taosArrayGetSize(rightRowLocations); ++i) { SRowLocation* rightRow = taosArrayGet(rightRowLocations, i); - int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightTagCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightTagKeyBuf); - SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen); + int32_t keyLen = fillKeyBufFromTagCols(pInfo->rightEqOnCondCols, rightRow->pDataBlock, rightRow->pos, pInfo->rightEqOnCondKeyBuf); + SArray** ppRows = tSimpleHashGet(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen); if (!ppRows) { SArray* rows = taosArrayInit(4, sizeof(SRowLocation)); taosArrayPush(rows, rightRow); - tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightTagKeyBuf, keyLen, &rows, POINTER_BYTES); + tSimpleHashPut(pInfo->rightBuildTable, pInfo->rightEqOnCondKeyBuf, keyLen, &rows, POINTER_BYTES); } else { taosArrayPush(*ppRows, rightRow); } @@ -466,8 +466,8 @@ static int32_t mergeJoinLeftRowsRightRows(SOperatorInfo* pOperator, SSDataBlock* SRowLocation* leftRow = taosArrayGet(leftRowLocations, i); SArray* pRightRows = NULL; if (useBuildTableTSRange) { - int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftTagCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftTagKeyBuf); - SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftTagKeyBuf, keyLen); + int32_t keyLen = fillKeyBufFromTagCols(pJoinInfo->leftEqOnCondCols, leftRow->pDataBlock, leftRow->pos, pJoinInfo->leftEqOnCondKeyBuf); + SArray** ppRightRows = tSimpleHashGet(pJoinInfo->rightBuildTable, pJoinInfo->leftEqOnCondKeyBuf, keyLen); if (!ppRightRows) { continue; } @@ -567,7 +567,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); - if (pJoinInfo->pTagEqualConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + if (pJoinInfo->pEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 91db527a02..c2771e0005 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,7 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); - CLONE_NODE_FIELD(pTagEqualConditions); + CLONE_NODE_FIELD(pEqualOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; @@ -588,7 +588,7 @@ static int32_t physiSysTableScanCopy(const SSystemTableScanPhysiNode* pSrc, SSys return TSDB_CODE_SUCCESS; } -static int32_t physiWindowCopy(const SWinodwPhysiNode* pSrc, SWinodwPhysiNode* pDst) { +static int32_t physiWindowCopy(const SWindowPhysiNode* pSrc, SWindowPhysiNode* pDst) { COPY_BASE_OBJECT_FIELD(node, physiNodeCopy); CLONE_NODE_LIST_FIELD(pExprs); CLONE_NODE_LIST_FIELD(pFuncs); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b4e2d95e26..d16cd79e97 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1432,7 +1432,7 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pTagEqualConditions); + code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pEqualOnConditions); } return code; } @@ -1451,7 +1451,7 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pTagEqualConditions); + code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pEqualOnConditions); } return code; } @@ -1905,7 +1905,7 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pTagEqualCondtions); + code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pEqualOnCondtions); } return code; } @@ -1930,7 +1930,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pTagEqualCondtions); + code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pEqualOnCondtions); } return code; } @@ -2135,7 +2135,7 @@ static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; static const char* jkWindowPhysiPlanMergeDataBlock = "MergeDataBlock"; static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { @@ -2176,7 +2176,7 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { } static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = jsonToPhysicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 45cebb4559..59b027d5ed 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2341,7 +2341,7 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pTagEqualCondtions); + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pEqualOnCondtions); } return code; } @@ -2372,7 +2372,7 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: - code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagEqualCondtions); + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEqualOnCondtions); break; default: break; @@ -2639,7 +2639,7 @@ enum { }; static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { - const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + const SWindowPhysiNode* pNode = (const SWindowPhysiNode*)pObj; int32_t code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); if (TSDB_CODE_SUCCESS == code) { @@ -2680,7 +2680,7 @@ static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { } static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { - SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + SWindowPhysiNode* pNode = (SWindowPhysiNode*)pObj; int32_t code = TSDB_CODE_SUCCESS; STlv* pTlv = NULL; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 67747497db..81b429e169 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -599,7 +599,7 @@ static void destroyPhysiNode(SPhysiNode* pNode) { nodesDestroyNode(pNode->pSlimit); } -static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { +static void destroyWinodwPhysiNode(SWindowPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pExprs); nodesDestroyList(pNode->pFuncs); @@ -1072,7 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); - nodesDestroyNode(pLogicNode->pTagEqualConditions); + nodesDestroyNode(pLogicNode->pEqualOnConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1205,7 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); - nodesDestroyNode(pPhyNode->pTagEqualCondtions); + nodesDestroyNode(pPhyNode->pEqualOnCondtions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { @@ -1243,7 +1243,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_FILL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: { @@ -1259,19 +1259,19 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: { SStateWinodwPhysiNode* pPhyNode = (SStateWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStateKey); break; } case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; - destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + destroyWinodwPhysiNode((SWindowPhysiNode*)pPhyNode); nodesDestroyNode(pPhyNode->pStartCond); nodesDestroyNode(pPhyNode->pEndCond); break; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 5be67389c8..1107389df9 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -740,18 +740,15 @@ static int32_t pushDownCondOptJoinExtractMergeCond(SOptimizeContext* pCxt, SJoin return code; } -static bool pushDownCondOptIsTag(SNode* pNode, SNodeList* pTableCols) { +static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) { if (QUERY_NODE_COLUMN != nodeType(pNode)) { return false; } SColumnNode* pCol = (SColumnNode*)pNode; - if (COLUMN_TYPE_TAG != pCol->colType) { - return false; - } return pushDownCondOptBelongThisTable(pNode, pTableCols); } -static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } @@ -770,22 +767,22 @@ static bool pushDownCondOptIsTagEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { } SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; - if (pushDownCondOptIsTag(pOper->pLeft, pLeftCols)) { - return pushDownCondOptIsTag(pOper->pRight, pRightCols); - } else if (pushDownCondOptIsTag(pOper->pLeft, pRightCols)) { - return pushDownCondOptIsTag(pOper->pRight, pLeftCols); + if (pushDownCondOptIsTableColumn(pOper->pLeft, pLeftCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pRightCols); + } else if (pushDownCondOptIsTableColumn(pOper->pLeft, pRightCols)) { + return pushDownCondOptIsTableColumn(pOper->pRight, pLeftCols); } return false; } -static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); int32_t code = TSDB_CODE_SUCCESS; SNodeList* pTagEqualConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - if (pushDownCondOptIsTagEqualCond(pJoin, pCond)) { + if (pushDownCondOptIsEqualOnCond(pJoin, pCond)) { code = nodesListMakeAppend(&pTagEqualConds, nodesCloneNode(pCond)); } } @@ -796,7 +793,7 @@ static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin } if (TSDB_CODE_SUCCESS == code) { - pJoin->pTagEqualConditions = pTempTagEqCond; + pJoin->pEqualOnConditions = pTempTagEqCond; return TSDB_CODE_SUCCESS; } else { nodesDestroyList(pTagEqualConds); @@ -805,18 +802,18 @@ static int32_t pushDownCondOptJoinExtractTagEqualLogicCond(SJoinLogicNode* pJoin return TSDB_CODE_SUCCESS; } -static int32_t pushDownCondOptJoinExtractTagEqualCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { - pJoin->pTagEqualConditions = NULL; + pJoin->pEqualOnConditions = NULL; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { - return pushDownCondOptJoinExtractTagEqualLogicCond(pJoin); + return pushDownCondOptJoinExtractEqualOnLogicCond(pJoin); } - if (pushDownCondOptIsTagEqualCond(pJoin, pJoin->pOnConditions)) { - pJoin->pTagEqualConditions = nodesCloneNode(pJoin->pOnConditions); + if (pushDownCondOptIsEqualOnCond(pJoin, pJoin->pOnConditions)) { + pJoin->pEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); } return TSDB_CODE_SUCCESS; @@ -857,7 +854,7 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p } if (TSDB_CODE_SUCCESS == code) { - code = pushDownCondOptJoinExtractTagEqualCond(pCxt, pJoin); + code = pushDownCondOptJoinExtractEqualOnCond(pCxt, pJoin); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e07c6ebcfe..a13ca6cba9 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,8 +705,8 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } - if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pTagEqualConditions) { - code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pTagEqualConditions, &pJoin->pTagEqualCondtions); + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pEqualOnConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pEqualOnConditions, &pJoin->pEqualOnCondtions); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); @@ -1150,7 +1150,7 @@ static int32_t createExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNo } } -static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWinodwPhysiNode* pWindow, +static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowPhysiNode* pWindow, SWindowLogicNode* pWindowLogicNode) { pWindow->triggerType = pWindowLogicNode->triggerType; pWindow->watermark = pWindowLogicNode->watermark; From f7ab8dabf086ef3740040635059aa36627bc1525 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 15:14:39 +0800 Subject: [PATCH 044/106] fix:[TD-23972] push subscribe msg to vnode even though consumer not change --- source/dnode/mnode/impl/src/mndSubscribe.c | 21 ++++++++++++---- source/dnode/vnode/src/tq/tq.c | 28 ++++++++++++---------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 2ae4ede25a..e62102fa77 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -133,10 +133,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg) { - if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { - terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; - return -1; - } +// if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { +// terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; +// return -1; +// } void *buf; int32_t tlen; @@ -269,6 +269,18 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { } } +static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ + for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ + SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); + SMqRebOutputVg outputVg = { + .oldConsumerId = pConsumerEp->consumerId, + .newConsumerId = pConsumerEp->consumerId, + .pVgEp = pVgEp, + }; + taosArrayPush(pOutput->rebVgs, &outputVg); + } +} + static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, int32_t imbConsumerNum) { const char *pSubKey = pOutput->pSub->key; @@ -318,6 +330,7 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas } } } + putNoTransferToOutput(pOutput, pConsumerEp); } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 41b43026ea..6f619e9bbe 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -569,20 +569,22 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); // atomic_store_32(&pHandle->epoch, 0); - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } - // remove if it has been register in the push manager, and return one empty block to consumer - taosWLockLatch(&pTq->lock); - tqUnregisterPushHandle(pTq, pHandle); - taosWUnLockLatch(&pTq->lock); - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } + + // kill executing task + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + } + + taosWLockLatch(&pTq->lock); + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } + // remove if it has been register in the push manager, and return one empty block to consumer + tqUnregisterPushHandle(pTq, pHandle); + taosWUnLockLatch(&pTq->lock); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } end: From f9da4abdcb495d0d9c75e4a6e9f14da47621a070 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 May 2023 15:38:52 +0800 Subject: [PATCH 045/106] fix: change more variable names --- include/libs/nodes/plannodes.h | 4 ++-- source/libs/executor/src/joinoperator.c | 22 +++++++++++----------- source/libs/nodes/src/nodesCloneFuncs.c | 2 +- source/libs/nodes/src/nodesCodeFuncs.c | 12 ++++++------ source/libs/nodes/src/nodesMsgFuncs.c | 4 ++-- source/libs/nodes/src/nodesUtilFuncs.c | 4 ++-- source/libs/planner/src/planOptimizer.c | 14 +++++++------- source/libs/planner/src/planPhysiCreater.c | 4 ++-- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f559ba2b51..a27237cab3 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -112,7 +112,7 @@ typedef struct SJoinLogicNode { SNode* pOnConditions; bool isSingleTableJoin; EOrder inputTsOrder; - SNode* pEqualOnConditions; + SNode* pColEqualOnConditions; } SJoinLogicNode; typedef struct SAggLogicNode { @@ -406,7 +406,7 @@ typedef struct SSortMergeJoinPhysiNode { SNode* pOnConditions; SNodeList* pTargets; EOrder inputTsOrder; - SNode* pEqualOnCondtions; + SNode* pColEqualOnConditions; } SSortMergeJoinPhysiNode; typedef struct SAggPhysiNode { diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 46c6b24295..442f8162ed 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -52,7 +52,7 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode* pCondAfterMerge; - SNode* pEqualOnConditions; + SNode* pColEqualOnConditions; SArray* leftEqOnCondCols; char* leftEqOnCondKeyBuf; @@ -117,13 +117,13 @@ static void extractEqualOnCondColsFromOper(SJoinOperatorInfo* pInfo, SOperatorIn } } -static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pTagEqualNode, +static void extractEqualOnCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pDownStream, SNode* pEqualOnCondNode, SArray* leftTagEqCols, SArray* rightTagEqCols) { SColumn left = {0}; SColumn right = {0}; - if (nodeType(pTagEqualNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pTagEqualNode)->condType == LOGIC_COND_TYPE_AND) { + if (nodeType(pEqualOnCondNode) == QUERY_NODE_LOGIC_CONDITION && ((SLogicConditionNode*)pEqualOnCondNode)->condType == LOGIC_COND_TYPE_AND) { SNode* pNode = NULL; - FOREACH(pNode, ((SLogicConditionNode*)pTagEqualNode)->pParameterList) { + FOREACH(pNode, ((SLogicConditionNode*)pEqualOnCondNode)->pParameterList) { SOperatorNode* pOperNode = (SOperatorNode*)pNode; extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); @@ -132,8 +132,8 @@ static void extractTagEqualCondCols(SJoinOperatorInfo* pInfo, SOperatorInfo** pD return; } - if (nodeType(pTagEqualNode) == QUERY_NODE_OPERATOR) { - SOperatorNode* pOperNode = (SOperatorNode*)pTagEqualNode; + if (nodeType(pEqualOnCondNode) == QUERY_NODE_OPERATOR) { + SOperatorNode* pOperNode = (SOperatorNode*)pEqualOnCondNode; extractEqualOnCondColsFromOper(pInfo, pDownStream, pOperNode, &left, &right); taosArrayPush(leftTagEqCols, &left); taosArrayPush(rightTagEqCols, &right); @@ -259,11 +259,11 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pInfo->pEqualOnConditions = pJoinNode->pEqualOnCondtions; - if (pInfo->pEqualOnConditions != NULL) { + pInfo->pColEqualOnConditions = pJoinNode->pColEqualOnConditions; + if (pInfo->pColEqualOnConditions != NULL) { pInfo->leftEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); pInfo->rightEqOnCondCols = taosArrayInit(4, sizeof(SColumn)); - extractTagEqualCondCols(pInfo, pDownstream, pInfo->pEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); + extractEqualOnCondCols(pInfo, pDownstream, pInfo->pColEqualOnConditions, pInfo->leftEqOnCondCols, pInfo->rightEqOnCondCols); initTagColskeyBuf(&pInfo->leftEqOnCondKeyLen, &pInfo->leftEqOnCondKeyBuf, pInfo->leftEqOnCondCols); initTagColskeyBuf(&pInfo->rightEqOnCondKeyLen, &pInfo->rightEqOnCondKeyBuf, pInfo->rightEqOnCondCols); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -309,7 +309,7 @@ static void mergeJoinDestoryBuildTable(SSHashObj* pBuildTable) { void destroyMergeJoinOperator(void* param) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; - if (pJoinOperator->pEqualOnConditions != NULL) { + if (pJoinOperator->pColEqualOnConditions != NULL) { mergeJoinDestoryBuildTable(pJoinOperator->rightBuildTable); taosMemoryFreeClear(pJoinOperator->rightEqOnCondKeyBuf); taosArrayDestroy(pJoinOperator->rightEqOnCondCols); @@ -567,7 +567,7 @@ static int32_t mergeJoinJoinDownstreamTsRanges(SOperatorInfo* pOperator, int64_t pJoinInfo->leftPos, timestamp, leftRowLocations, leftCreatedBlocks); mergeJoinGetDownStreamRowsEqualTimeStamp(pOperator, 1, pJoinInfo->rightCol.slotId, pJoinInfo->pRight, pJoinInfo->rightPos, timestamp, rightRowLocations, rightCreatedBlocks); - if (pJoinInfo->pEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { + if (pJoinInfo->pColEqualOnConditions != NULL && taosArrayGetSize(rightRowLocations) > 16) { mergeJoinFillBuildTable(pJoinInfo, rightRowLocations); rightUseBuildTable = true; taosArrayDestroy(rightRowLocations); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index c2771e0005..f18e666741 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,7 +401,7 @@ static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) { COPY_SCALAR_FIELD(joinType); CLONE_NODE_FIELD(pMergeCondition); CLONE_NODE_FIELD(pOnConditions); - CLONE_NODE_FIELD(pEqualOnConditions); + CLONE_NODE_FIELD(pColEqualOnConditions); COPY_SCALAR_FIELD(isSingleTableJoin); COPY_SCALAR_FIELD(inputTsOrder); return TSDB_CODE_SUCCESS; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index d16cd79e97..f061fe60d2 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1416,7 +1416,7 @@ static int32_t jsonToLogicPlan(const SJson* pJson, void* pObj) { static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanMergeCondition = "MergeConditions"; -static const char* jkJoinLogicPlanTagEqualConditions = "TagEqualConditions"; +static const char* jkJoinLogicPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; @@ -1432,7 +1432,7 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqualConditions, nodeToJson, pNode->pEqualOnConditions); + code = tjsonAddObject(pJson, jkJoinLogicPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); } return code; } @@ -1451,7 +1451,7 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqualConditions, &pNode->pEqualOnConditions); + code = jsonToNodeObject(pJson, jkJoinLogicPlanColEqualOnConditions, &pNode->pColEqualOnConditions); } return code; } @@ -1883,7 +1883,7 @@ static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkJoinPhysiPlanMergeCondition = "MergeCondition"; static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; static const char* jkJoinPhysiPlanTargets = "Targets"; -static const char* jkJoinPhysiPlanTagEqualConditions = "TagEqualConditions"; +static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions"; static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; @@ -1905,7 +1905,7 @@ static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkJoinPhysiPlanTagEqualConditions, nodeToJson, pNode->pEqualOnCondtions); + code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqualOnConditions); } return code; } @@ -1930,7 +1930,7 @@ static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkJoinPhysiPlanTagEqualConditions, &pNode->pEqualOnCondtions); + code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqualOnConditions); } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 59b027d5ed..e200b5dac5 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2341,7 +2341,7 @@ static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); } if (TSDB_CODE_SUCCESS == code) { - code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pEqualOnCondtions); + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqualOnConditions); } return code; } @@ -2372,7 +2372,7 @@ static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); break; case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS: - code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEqualOnCondtions); + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqualOnConditions); break; default: break; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 81b429e169..830d5886f4 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1072,7 +1072,7 @@ void nodesDestroyNode(SNode* pNode) { destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyNode(pLogicNode->pMergeCondition); nodesDestroyNode(pLogicNode->pOnConditions); - nodesDestroyNode(pLogicNode->pEqualOnConditions); + nodesDestroyNode(pLogicNode->pColEqualOnConditions); break; } case QUERY_NODE_LOGIC_PLAN_AGG: { @@ -1205,7 +1205,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pMergeCondition); nodesDestroyNode(pPhyNode->pOnConditions); nodesDestroyList(pPhyNode->pTargets); - nodesDestroyNode(pPhyNode->pEqualOnCondtions); + nodesDestroyNode(pPhyNode->pColEqualOnConditions); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 1107389df9..647a55a495 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -779,24 +779,24 @@ static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); int32_t code = TSDB_CODE_SUCCESS; - SNodeList* pTagEqualConds = NULL; + SNodeList* pEqualOnConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { if (pushDownCondOptIsEqualOnCond(pJoin, pCond)) { - code = nodesListMakeAppend(&pTagEqualConds, nodesCloneNode(pCond)); + code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond)); } } SNode* pTempTagEqCond = NULL; if (TSDB_CODE_SUCCESS == code) { - code = nodesMergeConds(&pTempTagEqCond, &pTagEqualConds); + code = nodesMergeConds(&pTempTagEqCond, &pEqualOnConds); } if (TSDB_CODE_SUCCESS == code) { - pJoin->pEqualOnConditions = pTempTagEqCond; + pJoin->pColEqualOnConditions = pTempTagEqCond; return TSDB_CODE_SUCCESS; } else { - nodesDestroyList(pTagEqualConds); + nodesDestroyList(pEqualOnConds); return TSDB_CODE_PLAN_INTERNAL_ERROR; } return TSDB_CODE_SUCCESS; @@ -804,7 +804,7 @@ static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { - pJoin->pEqualOnConditions = NULL; + pJoin->pColEqualOnConditions = NULL; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && @@ -813,7 +813,7 @@ static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJo } if (pushDownCondOptIsEqualOnCond(pJoin, pJoin->pOnConditions)) { - pJoin->pEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); + pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index a13ca6cba9..f11f1244b4 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -705,8 +705,8 @@ static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren pJoinLogicNode->pOnConditions, &pJoin->pOnConditions); } - if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pEqualOnConditions) { - code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pEqualOnConditions, &pJoin->pEqualOnCondtions); + if (TSDB_CODE_SUCCESS == code && NULL != pJoinLogicNode->pColEqualOnConditions) { + code = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pColEqualOnConditions, &pJoin->pColEqualOnConditions); } if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin); From d77eefdc93c7d6584eb033fb588b45714db66433 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 May 2023 15:43:31 +0800 Subject: [PATCH 046/106] fix: change more variable names --- source/libs/planner/src/planOptimizer.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 647a55a495..80beab3038 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -748,7 +748,7 @@ static bool pushDownCondOptIsTableColumn(SNode* pNode, SNodeList* pTableCols) { return pushDownCondOptBelongThisTable(pNode, pTableCols); } -static bool pushDownCondOptIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { +static bool pushDownCondOptIsColEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { if (QUERY_NODE_OPERATOR != nodeType(pCond)) { return false; } @@ -775,14 +775,14 @@ static bool pushDownCondOptIsEqualOnCond(SJoinLogicNode* pJoin, SNode* pCond) { return false; } -static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractColEqualOnLogicCond(SJoinLogicNode* pJoin) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(pJoin->pOnConditions); int32_t code = TSDB_CODE_SUCCESS; SNodeList* pEqualOnConds = NULL; SNode* pCond = NULL; FOREACH(pCond, pLogicCond->pParameterList) { - if (pushDownCondOptIsEqualOnCond(pJoin, pCond)) { + if (pushDownCondOptIsColEqualOnCond(pJoin, pCond)) { code = nodesListMakeAppend(&pEqualOnConds, nodesCloneNode(pCond)); } } @@ -802,17 +802,17 @@ static int32_t pushDownCondOptJoinExtractEqualOnLogicCond(SJoinLogicNode* pJoin) return TSDB_CODE_SUCCESS; } -static int32_t pushDownCondOptJoinExtractEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { +static int32_t pushDownCondOptJoinExtractColEqualOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { if (NULL == pJoin->pOnConditions) { pJoin->pColEqualOnConditions = NULL; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pJoin->pOnConditions))->condType) { - return pushDownCondOptJoinExtractEqualOnLogicCond(pJoin); + return pushDownCondOptJoinExtractColEqualOnLogicCond(pJoin); } - if (pushDownCondOptIsEqualOnCond(pJoin, pJoin->pOnConditions)) { + if (pushDownCondOptIsColEqualOnCond(pJoin, pJoin->pOnConditions)) { pJoin->pColEqualOnConditions = nodesCloneNode(pJoin->pOnConditions); } @@ -854,7 +854,7 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p } if (TSDB_CODE_SUCCESS == code) { - code = pushDownCondOptJoinExtractEqualOnCond(pCxt, pJoin); + code = pushDownCondOptJoinExtractColEqualOnCond(pCxt, pJoin); } if (TSDB_CODE_SUCCESS == code) { From be35194d169e77e97f524ff9e48e3c6c701fa58e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 19 May 2023 15:54:46 +0800 Subject: [PATCH 047/106] fix(query): add null check. --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index a196c2d398..8f168e67a8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -225,6 +225,9 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, void* tsdbCacherowsReaderClose(void* pReader) { SCacheRowsReader* p = pReader; + if (p == NULL) { + return NULL; + } if (p->pSchema != NULL) { for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) { From 6e6853413c0e4ce57dcd9355628cea2dc72b2fac Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 19 May 2023 16:48:05 +0800 Subject: [PATCH 048/106] fix: db cfg memory issue --- source/dnode/mnode/impl/src/mndDb.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e93b06fdea..0f7e3de2f5 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -929,7 +929,7 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) { cfgRsp->walRetentionSize = pDb->cfg.walRetentionSize; cfgRsp->walSegmentSize = pDb->cfg.walSegmentSize; cfgRsp->numOfRetensions = pDb->cfg.numOfRetensions; - cfgRsp->pRetensions = pDb->cfg.pRetensions; + cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); cfgRsp->schemaless = pDb->cfg.schemaless; cfgRsp->sstTrigger = pDb->cfg.sstTrigger; } @@ -972,6 +972,8 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { _OVER: + tFreeSDbCfgRsp(&cfgRsp); + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } From 3dbbe6309248c10e28cd7c2097f9d61f04d82b35 Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Fri, 19 May 2023 16:50:27 +0800 Subject: [PATCH 049/106] fix: select last(*) ..group by tag return wrong rows of data --- source/libs/planner/src/planOptimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 5be67389c8..5d0757883c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2377,7 +2377,7 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (NULL != cxt.pLastCols) { cxt.doAgg = false; lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols); - NODES_DESTORY_LIST(pScan->pScanPseudoCols); + nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt); lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols); nodesClearList(cxt.pLastCols); } From ff2b545b27afa6f28f704d896910c04e727d19c8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 17:16:47 +0800 Subject: [PATCH 050/106] fix:set task status killed when vnode receive subscribe msg from mnode --- include/libs/executor/executor.h | 2 ++ source/dnode/vnode/src/tq/tq.c | 4 ++-- source/libs/executor/src/executor.c | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index b7e6c42e3b..2598b5c28c 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,6 +90,8 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); + int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); // todo refactor diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6f619e9bbe..9e43c4a944 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -394,7 +394,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); - + qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; @@ -574,7 +574,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // kill executing task qTaskInfo_t pTaskInfo = pHandle->execHandle.task; if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); } taosWLockLatch(&pTq->lock); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index c4a56d78ae..eea542e042 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,6 +180,11 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { + SExecTaskInfo* pTaskInfo = tinfo; + pTaskInfo->code = code; +} + int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { return TSDB_CODE_APP_ERROR; From c7c255e967aa250741bcb839f329634a99834b29 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 18:06:05 +0800 Subject: [PATCH 051/106] fix:set task status killed when vnode receive subscribe msg from mnode --- source/dnode/vnode/src/tq/tq.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9e43c4a944..b1e412d5ec 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -370,6 +370,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); + qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; @@ -394,7 +395,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); - qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqSetHandleIdle(pHandle); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; @@ -571,15 +571,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // atomic_store_32(&pHandle->epoch, 0); } - // kill executing task - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); - } - taosWLockLatch(&pTq->lock); - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); + // kill executing task + if(tqIsHandleExec(pHandle)) { + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); + } + + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); From 33d6df8717875f1f2001d8ea8dde8339beee279d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 19:11:49 +0800 Subject: [PATCH 052/106] fix:set task status killed when vnode receive subscribe msg from mnode --- source/dnode/vnode/src/tq/tq.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b1e412d5ec..d9961202b1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -367,6 +367,20 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } + // 3. update the epoch value + int32_t savedEpoch = pHandle->epoch; + if (savedEpoch <= reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, + reqEpoch); + pHandle->epoch = reqEpoch; + }else { + tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", + consumerId, TD_VID(pTq->pVnode), req.subKey, savedEpoch, reqEpoch); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosWUnLockLatch(&pTq->lock); + return -1; + } + bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); @@ -381,14 +395,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } - // 3. update the epoch value - int32_t savedEpoch = pHandle->epoch; - if (savedEpoch < reqEpoch) { - tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, - reqEpoch); - pHandle->epoch = reqEpoch; - } - char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, @@ -396,6 +402,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { code = tqExtractDataForMq(pTq, pHandle, &req, pMsg); tqSetHandleIdle(pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, , set handle idle, pHandle:%p", consumerId, vgId, req.subKey, pHandle); return code; } @@ -561,17 +568,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); goto end; } else { + taosWLockLatch(&pTq->lock); + if (pHandle->consumerId == req.newConsumerId) { // do nothing tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs, should not reach here", req.vgId, req.newConsumerId); -// atomic_add_fetch_32(&pHandle->epoch, 1); } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); -// atomic_store_32(&pHandle->epoch, 0); } + atomic_add_fetch_32(&pHandle->epoch, 1); - taosWLockLatch(&pTq->lock); // kill executing task if(tqIsHandleExec(pHandle)) { qTaskInfo_t pTaskInfo = pHandle->execHandle.task; From 1ddbdad33cf487e94d2edf6a3138324f123723e5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 21:40:33 +0800 Subject: [PATCH 053/106] fix:set task status killed when vnode receive subscribe msg from mnode --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d9961202b1..da366c1610 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -362,7 +362,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (pHandle->consumerId != consumerId) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + terrno = TSDB_CODE_TMQ_INVALID_MSG; taosWUnLockLatch(&pTq->lock); return -1; } From f8de38e5327ca92025c6f7a00fa6e02c08b6e710 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 May 2023 22:52:58 +0800 Subject: [PATCH 054/106] fix:core dump in tsdbreader is null when tsdbreader is closed by subscribe msg and current offset equal to saved --- source/libs/executor/src/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index eea542e042..7d251fb074 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1080,7 +1080,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset) && pOffset->type != TMQ_OFFSET__SNAPSHOT_DATA) { return 0; } From 04858fae22b4b2f0fa6207b56e84dab0fc679ea6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 May 2023 12:20:54 +0800 Subject: [PATCH 055/106] fix:tsdbreader is free by mistake --- include/libs/executor/executor.h | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/dnode/vnode/src/tq/tq.c | 8 ++++---- source/dnode/vnode/src/tq/tqScan.c | 12 ++++++++---- source/libs/executor/src/executor.c | 10 +++++----- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 2598b5c28c..91db78e9cd 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,7 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index a318b9886e..922a85c094 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -75,7 +75,7 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, terrstr(code)); + dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, tstrerror(code)); vmSendRsp(pMsg, code); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index da366c1610..ce52277d71 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -384,7 +384,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); - qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); +// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; @@ -586,9 +586,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); } - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pTaskInfo); - } +// if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { +// qStreamCloseTsdbReader(pTaskInfo); +// } } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index e4b2fa8821..2f470e2221 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -84,8 +84,10 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs qStreamSetOpen(task); tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); - if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { - tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); + code = qExecTask(task, &pDataBlock, &ts); + if (code != TSDB_CODE_SUCCESS) { + tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code)); + terrno = code; return -1; } @@ -128,8 +130,10 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; tqDebug("tmqsnap task start to execute"); - if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + int code = qExecTask(task, &pDataBlock, &ts); + if (code != 0) { + tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, tstrerror(code)); + terrno = code; return -1; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7d251fb074..a73deffa52 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,10 +180,10 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { - SExecTaskInfo* pTaskInfo = tinfo; - pTaskInfo->code = code; -} +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { +// SExecTaskInfo* pTaskInfo = tinfo; +// pTaskInfo->code = code; +//} int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { @@ -1080,7 +1080,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset) && pOffset->type != TMQ_OFFSET__SNAPSHOT_DATA) { + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } From 0b12224021345631b32ee889baddfaf13efcab28 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 May 2023 14:01:12 +0800 Subject: [PATCH 056/106] test: add schemless insert case --- .../system-test/eco-system/schemaless/insert.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/system-test/eco-system/schemaless/insert.py b/tests/system-test/eco-system/schemaless/insert.py index def70748d7..901196ebfd 100644 --- a/tests/system-test/eco-system/schemaless/insert.py +++ b/tests/system-test/eco-system/schemaless/insert.py @@ -104,34 +104,38 @@ class TDTestCase: tdLog.info(f" exe failed. i={i} {sql}") traceback.print_exc() + def genTags(self, i): + tags = f"t1={i},t2=abc,t3=work" + return tags # change table schema def schemaless_insert(self, change_cnt): # init ts = 1683194263000 for i in range(change_cnt): - t1 = i % 1000 index = int(i/10000) % 600 cols = self.genCols("c", 5, index, False) - tags = f"t1={t1},t2=abc,t3=work" - sql = f'sta,{tags} {cols} {ts + i}' + tags = self.genTags(index) + sql = f'{self.stable},{tags} {cols} {ts + i}' self.insert(sql, i) # run + def run(self): # seed #random.seed(int(time.time())) - self.dbname = "schema_change" + self.dbname = "eco_system" + self.stable = "sml_stb" # switch db tdSql.execute(f"use {self.dbname};") - tdSql.execute(f"drop table if exists sta;") + tdSql.execute(f"drop table if exists {self.stable};") # change meters try: - self.schemaless_insert(1000) + self.schemaless_insert(1000000) except: traceback.print_exc() From 8183be4afff416a66d9ec07879ac5e620d1e43d9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 May 2023 16:13:52 +0800 Subject: [PATCH 057/106] fix:move sleep logic from rpc thread to tmq thread & fix some error --- source/client/src/clientTmq.c | 9 ++++++--- source/dnode/vnode/src/tq/tq.c | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c08fbd0adf..b57ecfa845 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1275,7 +1275,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { // in case of consumer mismatch, wait for 500ms and retry if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { - taosMsleep(500); +// taosMsleep(500); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, wait for 500ms and set status to be RECOVER", tmq->consumerId); @@ -1289,8 +1289,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; taosWriteQitem(tmq->mqueue, pRspWrapper); - } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert - taosMsleep(500); +// } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert +// taosMsleep(5); } else{ tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, vgId, epoch, tstrerror(code), requestId); @@ -1731,6 +1731,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p // broadcast the poll request to all related vnodes static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { + if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){ + return 0; + } int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ce52277d71..91524f7a95 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -360,22 +360,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { // 2. check re-balance status if (pHandle->consumerId != consumerId) { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); - terrno = TSDB_CODE_TMQ_INVALID_MSG; + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; taosWUnLockLatch(&pTq->lock); return -1; } // 3. update the epoch value - int32_t savedEpoch = pHandle->epoch; - if (savedEpoch <= reqEpoch) { - tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, - reqEpoch); - pHandle->epoch = reqEpoch; - }else { - tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", - consumerId, TD_VID(pTq->pVnode), req.subKey, savedEpoch, reqEpoch); + if (pHandle->epoch > reqEpoch) { + tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->epoch, reqEpoch); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; taosWUnLockLatch(&pTq->lock); return -1; @@ -395,6 +390,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } + if (pHandle->epoch < reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, pHandle->epoch, reqEpoch); + pHandle->epoch = reqEpoch; + } + char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, @@ -577,7 +577,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId); } - atomic_add_fetch_32(&pHandle->epoch, 1); +// atomic_add_fetch_32(&pHandle->epoch, 1); // kill executing task if(tqIsHandleExec(pHandle)) { From 7b735c67d0a84f001c41c8f1145655cd3ddb40b4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 20 May 2023 16:36:25 +0800 Subject: [PATCH 058/106] fix:move sleep logic from rpc thread to tmq thread & fix some error --- source/dnode/vnode/src/tq/tq.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 91524f7a95..db03e97556 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -367,15 +367,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } - // 3. update the epoch value - if (pHandle->epoch > reqEpoch) { - tqError("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, savedEpoch:%d > reqEpoch:%d ", - consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->epoch, reqEpoch); - terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - taosWUnLockLatch(&pTq->lock); - return -1; - } - bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); @@ -390,6 +381,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { taosMsleep(10); } + // 3. update the epoch value if (pHandle->epoch < reqEpoch) { tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, pHandle->epoch, reqEpoch); pHandle->epoch = reqEpoch; From c9add944209c4e314ce54bf0f8975e4f8bdc7784 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 May 2023 09:58:52 +0800 Subject: [PATCH 059/106] docs: update readme main (#21401) * docs: update readme with libgflags * docs: update README-CN.md --- README-CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-CN.md b/README-CN.md index a6dfefc47a..cee6f97e9a 100644 --- a/README-CN.md +++ b/README-CN.md @@ -15,7 +15,7 @@ [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -简体中文 | [English](README.md) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/) +简体中文 | [English](README.md) | [TDengine 云服务](https://cloud.taosdata.com/?utm_medium=cn&utm_source=github) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/) # TDengine 简介 From 28d562a55d14d8e7dfb6b19e3081937fab3c239a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 22 May 2023 03:56:11 +0000 Subject: [PATCH 060/106] add parameter --- docs/en/14-reference/12-config/index.md | 19 +++++++++++++++++++ docs/zh/14-reference/12-config/index.md | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 430487a3af..cb9bebd723 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -97,6 +97,25 @@ The parameters described in this document by the effect that they have on the sy | Value Range | 10-50000000 | | Default Value | 5000 | +### numOfRpcSessions + +| Attribute | Description | +| ------------- | ---------------------------------------------------- | +| Applicable | Client/Server | +| Meaning | The maximum number of connection to create | +| Value Range | 100-100000 | +| Default Value | 10000 | + +### timeToGetAvailableConn + +| Attribute | Description | +| ------------- | ---------------------------------------------------- | +| Applicable | Client/Server | +| Meaning | The maximum waiting time to get avaliable conn | +| Value Range | 10-50000000(ms) | +| Default Value | 500000 | + + ## Monitoring Parameters :::note diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index e5efd77f80..9533f82ed8 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -97,6 +97,25 @@ taos --dump-config | 取值范围 | 10-50000000 | | 缺省值 | 5000 | +### numOfRpcSessions + +| 属性 | 说明 | +| -------- | ----------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 一个客户端能创建的最大连接数 | +| 取值范围 | 100-100000 | +| 缺省值 | 10000 | + +### timeToGetAvailableConn + +| 属性 | 说明 | +| -------- | ----------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 |获得可用连接的最长等待时间 | +| 取值范围 | 10-50000000(单位为毫秒) | +| 缺省值 | 500000 | + + ## 监控相关 :::note From 591b51f624139db99825be975199c25c7d094c2e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 May 2023 12:42:21 +0800 Subject: [PATCH 061/106] fix:set task code success after kill --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index db03e97556..f2956ea488 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -575,7 +575,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg if(tqIsHandleExec(pHandle)) { qTaskInfo_t pTaskInfo = pHandle->execHandle.task; if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); } // if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { From f67caaed1f5359ab806933603d40c41729e7ee24 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 22 May 2023 06:09:55 +0000 Subject: [PATCH 062/106] opt compile --- docs/zh/14-reference/12-config/index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 9533f82ed8..90070098b8 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -91,20 +91,20 @@ taos --dump-config ### maxShellConns | 属性 | 说明 | -| -------- | ----------------------- | +| --------| ----------------------- | | 适用范围 | 仅服务端适用 | -| 含义 | 一个 dnode 容许的连接数 | +| 含义 | 一个 dnode 容许的连接数 | | 取值范围 | 10-50000000 | -| 缺省值 | 5000 | +| 缺省值 | 5000 | ### numOfRpcSessions -| 属性 | 说明 | -| -------- | ----------------------- | -| 适用范围 | 客户端和服务端都适用 | -| 含义 | 一个客户端能创建的最大连接数 | +| 属性 | 说明 | +| --------| ---------------------- | +| 适用范围 | 客户端和服务端都适用 | +| 含义 | 一个客户端能创建的最大连接数| | 取值范围 | 100-100000 | -| 缺省值 | 10000 | +| 缺省值 | 10000 | ### timeToGetAvailableConn From d6b188f15d36a506a95ce3bf663700da9a61bc0c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 22 May 2023 06:10:00 +0000 Subject: [PATCH 063/106] opt compile --- docs/zh/14-reference/12-config/index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 90070098b8..ed6c9b4505 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -99,21 +99,21 @@ taos --dump-config ### numOfRpcSessions -| 属性 | 说明 | +| 属性 | 说明 | | --------| ---------------------- | | 适用范围 | 客户端和服务端都适用 | | 含义 | 一个客户端能创建的最大连接数| | 取值范围 | 100-100000 | -| 缺省值 | 10000 | +| 缺省值 | 10000 | ### timeToGetAvailableConn -| 属性 | 说明 | -| -------- | ----------------------- | -| 适用范围 | 客户端和服务端都适用 | -| 含义 |获得可用连接的最长等待时间 | -| 取值范围 | 10-50000000(单位为毫秒) | -| 缺省值 | 500000 | +| 属性 | 说明 | +| -------- | --------------------| +| 适用范围 | 客户端和服务端都适用 | +| 含义 |获得可用连接的最长等待时间| +| 取值范围 | 10-50000000(单位为毫秒)| +| 缺省值 | 500000 | ## 监控相关 From ae515851c4d753162f5bcd10aa940658e377055f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 May 2023 15:32:30 +0800 Subject: [PATCH 064/106] fix:remove get schema from previous for taosx --- source/dnode/vnode/src/meta/metaQuery.c | 38 ------------------------- 1 file changed, 38 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d464f64de3..428d201a17 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -387,42 +387,6 @@ _query: tDecoderClear(&dc); goto _exit; } - { // Traverse to find the previous qualified data - TBC *pCur; - tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); - STbDbKey key = {.version = sver, .uid = INT64_MAX}; - int c = 0; - tdbTbcMoveTo(pCur, &key, sizeof(key), &c); - if (c < 0) { - tdbTbcMoveToPrev(pCur); - } - - void *pKey = NULL; - void *pVal = NULL; - int vLen = 0, kLen = 0; - while (1) { - int32_t ret = tdbTbcPrev(pCur, &pKey, &kLen, &pVal, &vLen); - if (ret < 0) break; - - STbDbKey *tmp = (STbDbKey *)pKey; - if (tmp->uid != uid) { - continue; - } - SDecoder dcNew = {0}; - SMetaEntry meNew = {0}; - tDecoderInit(&dcNew, pVal, vLen); - metaDecodeEntry(&dcNew, &meNew); - pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow); - tDecoderClear(&dcNew); - tdbTbcClose(pCur); - tdbFree(pKey); - tdbFree(pVal); - goto _exit; - } - tdbFree(pKey); - tdbFree(pVal); - tdbTbcClose(pCur); - } } else if (me.type == TSDB_CHILD_TABLE) { uid = me.ctbEntry.suid; tDecoderClear(&dc); @@ -447,7 +411,6 @@ _query: tDecoderClear(&dc); _exit: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } @@ -455,7 +418,6 @@ _exit: return pSchema; _err: - tDecoderClear(&dc); if (lock) { metaULock(pMeta); } From 7b20d2fa7d63af8959acaf29892a892aece67b71 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 22 May 2023 20:10:13 +0800 Subject: [PATCH 065/106] update crash_gen --- tests/pytest/crash_gen/crash_gen_main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index ec588659e9..9228cecdb5 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1722,12 +1722,14 @@ class TaskCreateDb(StateTransitionTask): vg_nums = random.randint(1, 8) cache_model = Dice.choice(['none', 'last_row', 'last_value', 'both']) buffer = random.randint(3, 128) + walRetentionPeriod = random.randint(1, 10000) dbName = self._db.getName() - self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} ".format(dbName, repStr, + self.execWtSql(wt, "create database {} {} {} vgroups {} cachemodel '{}' buffer {} wal_retention_period {} ".format(dbName, repStr, updatePostfix, vg_nums, cache_model, - buffer)) + buffer, + walRetentionPeriod)) if dbName == "db_0" and Config.getConfig().use_shadow_db: self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix)) From d8387b95ff13682b3706bd9fbb06cf8094596b32 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 May 2023 20:14:43 +0800 Subject: [PATCH 066/106] fix:set task status to 0 in inti --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/executor/src/executor.c | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index f5f7218ab9..fe949d92ce 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,7 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); -//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f2956ea488..85d8d073f7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -370,7 +370,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); -// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); + qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a73deffa52..eea542e042 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,10 +180,10 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } -//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { -// SExecTaskInfo* pTaskInfo = tinfo; -// pTaskInfo->code = code; -//} +void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { + SExecTaskInfo* pTaskInfo = tinfo; + pTaskInfo->code = code; +} int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { From ca479f57dfd40dc57503a33065f1798c7ff875fb Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Mon, 22 May 2023 21:40:38 +0800 Subject: [PATCH 067/106] Remove taosx, connector, explorer and taoskeeper in cloud tdengine --- packaging/docker/dockerbuild.sh | 10 ++++++++++ packaging/tools/makeclient.sh | 4 ++-- packaging/tools/makepkg.sh | 31 ++++++++++++++++++------------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 4b6fc8576b..ee09854176 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -123,6 +123,16 @@ else echo "Unknown cpuType: ${cpuType}" exit 1 fi +# check the tdengine cloud base image existed or not +if [ "$cloudBuild" == "y" ]; then + CloudBase=$(docker images | grep tdengine/tdengine-cloud-base ||:) + if [[ "$CloudBase" == "" ]]; then + echo "Rebuild tdengine cloud base image..." + docker build --rm -f "${communityDir}/packaging/docker/DockerfileCloud.base" -t tdengine/tdengine-cloud-base "." --build-arg cpuType=${cpuTypeAlias} + else + echo "Already found tdengine cloud base image" + fi +fi docker build --rm -f "${Dockerfile}" --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias} if [ "$cloudBuild" != "y" ]; then diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 28dc770755..c4b074f903 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -175,7 +175,7 @@ if [ "$pagMode" == "lite" ]; then fi chmod a+x ${install_dir}/install_client.sh -if [[ $productName == "TDengine" ]]; then +if [[ $productName == "TDengine" ]] && [ "$verMode" != "cloud" ]; then # Copy example code mkdir -p ${install_dir}/examples examples_dir="${top_dir}/examples" @@ -191,7 +191,7 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json fi - if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then + if [ "$verMode" == "cluster" ]; then # Copy connector connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 3da005c405..7bf81d2807 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -69,25 +69,30 @@ if [ "$pagMode" == "lite" ]; then bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark " taostools_bin_files="" else - - wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ + if [ "$verMode" == "cloud" ]; then + taostools_bin_files=" ${build_dir}/bin/taosdump \ + ${build_dir}/bin/taosBenchmark" + else + wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ || echo "failed to download TDinsight.sh" - # download TDinsight caches - orig_pwd=$(pwd) - tdinsight_caches="" - cd ${build_dir}/bin/ && \ - chmod +x TDinsight.sh - ./TDinsight.sh --download-only ||: -# tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ") - cd $orig_pwd - echo "TDinsight caches: $tdinsight_caches" + # download TDinsight caches + orig_pwd=$(pwd) + tdinsight_caches="" + cd ${build_dir}/bin/ && \ + chmod +x TDinsight.sh + ./TDinsight.sh --download-only ||: + # tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ") + cd $orig_pwd + echo "TDinsight caches: $tdinsight_caches" - taostools_bin_files=" ${build_dir}/bin/taosdump \ + taostools_bin_files=" ${build_dir}/bin/taosdump \ ${build_dir}/bin/taosBenchmark \ ${build_dir}/bin/TDinsight.sh \ ${build_dir}/bin/tdengine-datasource.zip \ ${build_dir}/bin/tdengine-datasource.zip.md5sum" + fi + [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer') @@ -334,7 +339,7 @@ mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : # Copy connector -if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then +if [ "$verMode" == "cluster" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then From ed612088c05e40880520a1396f630131d51bd49a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 23 May 2023 10:32:13 +0800 Subject: [PATCH 068/106] fix:lost data because tsdbreader or task is killed --- include/libs/executor/executor.h | 2 +- source/dnode/vnode/src/tq/tq.c | 14 +++++++------- source/libs/executor/src/executor.c | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index fe949d92ce..f5f7218ab9 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -90,7 +90,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 */ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 85d8d073f7..17eac7d096 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -370,7 +370,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { bool exec = tqIsHandleExec(pHandle); if(!exec) { tqSetHandleExec(pHandle); - qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); +// qSetTaskCode(pHandle->execHandle.task, TDB_CODE_SUCCESS); tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, set handle exec, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosWUnLockLatch(&pTq->lock); break; @@ -572,16 +572,16 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // atomic_add_fetch_32(&pHandle->epoch, 1); // kill executing task - if(tqIsHandleExec(pHandle)) { - qTaskInfo_t pTaskInfo = pHandle->execHandle.task; - if (pTaskInfo != NULL) { - qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); - } +// if(tqIsHandleExec(pHandle)) { +// qTaskInfo_t pTaskInfo = pHandle->execHandle.task; +// if (pTaskInfo != NULL) { +// qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); +// } // if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { // qStreamCloseTsdbReader(pTaskInfo); // } - } +// } // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); taosWUnLockLatch(&pTq->lock); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index eea542e042..a73deffa52 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -180,10 +180,10 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { doSetTaskId(pTaskInfo->pRoot); } -void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { - SExecTaskInfo* pTaskInfo = tinfo; - pTaskInfo->code = code; -} +//void qSetTaskCode(qTaskInfo_t tinfo, int32_t code) { +// SExecTaskInfo* pTaskInfo = tinfo; +// pTaskInfo->code = code; +//} int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { From 00524b3185f6eac4cec7148ccc4fd98d7cce5341 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Tue, 23 May 2023 11:36:42 +0800 Subject: [PATCH 069/106] remove the taosdump in cloud --- packaging/tools/makepkg.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 7bf81d2807..fc02a288ca 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -70,8 +70,7 @@ if [ "$pagMode" == "lite" ]; then taostools_bin_files="" else if [ "$verMode" == "cloud" ]; then - taostools_bin_files=" ${build_dir}/bin/taosdump \ - ${build_dir}/bin/taosBenchmark" + taostools_bin_files="${build_dir}/bin/taosBenchmark" else wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ From 13a6eb177c3c9682d2c269e0143d4065e4b732a6 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Tue, 23 May 2023 11:38:04 +0800 Subject: [PATCH 070/106] add space before the bin files --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index fc02a288ca..748b79bd7f 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -70,7 +70,7 @@ if [ "$pagMode" == "lite" ]; then taostools_bin_files="" else if [ "$verMode" == "cloud" ]; then - taostools_bin_files="${build_dir}/bin/taosBenchmark" + taostools_bin_files=" ${build_dir}/bin/taosBenchmark" else wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \ && echo "TDinsight.sh downloaded!" \ From a54eaa20ffe37bcf6ffd59291ed0b52c530f78b5 Mon Sep 17 00:00:00 2001 From: xleili Date: Tue, 23 May 2023 18:11:00 +0800 Subject: [PATCH 071/106] build: release ver-3.0.4.2 --- cmake/cmake.version | 2 +- packaging/tools/makepkg.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 3166a0695c..1d43986de4 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.4.1") + SET(TD_VER_NUMBER "3.0.4.2") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 748b79bd7f..e297902bf9 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -345,7 +345,7 @@ if [ "$verMode" == "cluster" ]; then tmp_pwd=`pwd` cd ${install_dir}/connector if [ ! -d taos-connector-jdbc ];then - git clone -b 3.1.0 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + git clone -b 3.2.1 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: fi cd taos-connector-jdbc mvn clean package -Dmaven.test.skip=true From 34cf6aad3014516cb6bd69ea4797bc5a08295085 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 23 May 2023 18:29:53 +0800 Subject: [PATCH 072/106] fix: alter dnode option has no effect --- source/dnode/mnode/impl/src/mndDnode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 1f58ae97a3..e5ee4b8a21 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -965,7 +965,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); mInfo("dnode:%d, send config req to dnode, app:%p config:%s value:%s", cfgReq.dnodeId, pReq->info.ahandle, dcfgReq.config, dcfgReq.value); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .info = pReq->info}; + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; tmsgSendReq(&epSet, &rpcMsg); code = 0; } From 5bedf6b19ae337f77a43941ae6bc58e153d26517 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 23 May 2023 19:21:52 +0800 Subject: [PATCH 073/106] enh: support config batch rows number when import data from csv file --- include/common/tglobal.h | 2 +- source/common/src/tglobal.c | 10 +++---- source/libs/parser/src/parInsertSql.c | 4 ++- source/libs/parser/src/parInsertUtil.c | 37 +++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 6fde7b48a2..44276102cf 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -122,7 +122,7 @@ extern bool tsUseAdapter; // client extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; -extern int32_t tsMaxMemUsedByInsert; +extern int32_t tsMaxInsertBatchRows; // build info extern char version[]; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index b85dfa80b0..bef63d8a49 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -148,8 +148,8 @@ int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 database precision unit for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; -// maximum memory allowed to be allocated for a single csv load (in MB) -int32_t tsMaxMemUsedByInsert = 1024; +// maximum batch rows numbers imported from a single csv load +int32_t tsMaxInsertBatchRows = 1000000; float tsSelectivityRatio = 1.0; int32_t tsTagFilterResCacheSize = 1024 * 10; @@ -340,7 +340,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; - if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; + if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; @@ -725,7 +725,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { // tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; // tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; @@ -987,7 +987,7 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { - tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; + tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; } else if (strcasecmp("maxRetryWaitTime", name) == 0) { tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 02de9f227d..6751b34d8e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1552,7 +1552,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxInsertBatchRows) { pStmt->fileProcessing = true; break; } @@ -1561,6 +1561,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } taosMemoryFree(pLine); + parserDebug("0x%" PRIx64 " %d rows have been parsed", pCxt->pComCxt->requestId, *pNumOfRows); + if (TSDB_CODE_SUCCESS == code && 0 == (*pNumOfRows) && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) && !pStmt->fileProcessing) { code = buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index f921094752..952a927136 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -272,6 +272,41 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat return code; } +static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { + int32_t code = TSDB_CODE_SUCCESS; + SSubmitTbData* pTmp = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pTmp) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTmp->flags = pSrc->flags; + pTmp->suid = pSrc->suid; + pTmp->uid = pSrc->uid; + pTmp->sver = pSrc->sver; + pTmp->pCreateTbReq = pSrc->pCreateTbReq; + if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + pTmp->aCol = taosArrayInit(128, sizeof(SColData)); + if (NULL == pTmp->aCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } else { + pTmp->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pTmp->aRowP) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pTmp); + } + } + } + + taosMemoryFree(pSrc); + if (TSDB_CODE_SUCCESS == code) { + *pDst = pTmp; + } + + return code; +} + + static void resetColValues(SArray* pValues) { int32_t num = taosArrayGetSize(pValues); for (int32_t i = 0; i < num; ++i) { @@ -381,7 +416,7 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx } } taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); - taosMemoryFreeClear(pTableCxt->pData); + rebuildTableData(pTableCxt->pData, &pTableCxt->pData); qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); From b52f867a1f933772765b44d6b9a4297cc489a7fc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 24 May 2023 09:56:59 +0800 Subject: [PATCH 074/106] fix: rebuild table data issue --- source/libs/parser/src/parInsertUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 952a927136..de7d154db6 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -282,7 +282,7 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { pTmp->suid = pSrc->suid; pTmp->uid = pSrc->uid; pTmp->sver = pSrc->sver; - pTmp->pCreateTbReq = pSrc->pCreateTbReq; + pTmp->pCreateTbReq = NULL; if (pTmp->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { pTmp->aCol = taosArrayInit(128, sizeof(SColData)); if (NULL == pTmp->aCol) { From 9a421a92f90aefb83c54f870fe18c672a3bd904e Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Wed, 24 May 2023 10:32:10 +0800 Subject: [PATCH 075/106] fix: core dump when select with interval caused by order --- source/libs/executor/src/filloperator.c | 2 +- source/libs/executor/src/tfill.c | 2 +- tests/script/tsim/query/interval.sim | 36 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 73222ee375..6d4b8570d1 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -258,7 +258,7 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t STimeWindow w = {0}; int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - getInitialStartTimeWindow(pInterval, startKey, &w, order); + getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index c98746c241..a39521f25c 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -561,7 +561,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma ASSERT(numOfRes >= numOfRows); } else { // reach the end of data if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || - (ekey1 >= pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { + (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 833da4a8ba..e2b0d219cb 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -170,6 +170,42 @@ endi # return -1 #endi +print ================ step10 +print -------- create database and table +sql create database if not exists test +sql use test +sql create stable st (ts timestamp, c2 int) tags(tg int) +print -------- insert 300 rows data +$i = 0 +while $i < 300 + $t = 1577807983000 + $cc = $i * 1000 + $t = $t + $cc + sql select $i % 3 + if $data00 != 0.000000000 then + goto continue_while + endi + sql select $i % 4 + if $data00 == 0.000000000 then + goto continue_while + endi + sql insert into t1 using st tags(1) values ( $t , $i ) + continue_while: + $i = $i + 1 +endw + +$ms1 = 1577808120000 +$ms2 = 1577808000000 +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts asc +print ----> select asc rows: $rows +$asc_rows = $rows +sql select * from (select _wstart, last(ts) as ts, avg(c2) as av from t1 where ts <= $ms1 and ts >= $ms2 interval(10s) sliding(1s) fill(NULL)) order by ts desc +print ----> select desc rows: $rows +$desc_rows = $rows +if $desc_rows != $asc_rows then + return -1 +endi + print =============== clear #sql drop database $db #sql select * from information_schema.ins_databases From d6d0bf0e41ef5a0490376d2ce8658826868b06f9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 24 May 2023 10:47:26 +0800 Subject: [PATCH 076/106] enh: add varchar sma --- source/common/src/tdataformat.c | 52 ++++++++++++++++--- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index b18bd882ae..55204045ba 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2441,7 +2441,7 @@ _exit: int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, char *data) { int32_t code = 0; - if(data == NULL){ + if (data == NULL) { for (int32_t i = 0; i < nRows; ++i) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); } @@ -2455,8 +2455,9 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; } else { - if(ASSERT(varDataTLen(data + offset) <= bytes)){ - uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes); + if (ASSERT(varDataTLen(data + offset) <= bytes)) { + uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), + bytes); code = TSDB_CODE_INVALID_PARA; goto _exit; } @@ -2508,7 +2509,7 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { ASSERT(pColData->type == pBind->buffer_type); } - + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { @@ -3521,6 +3522,43 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum } } +static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = 0; + *numOfNull = 0; + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + case (HAS_NONE | HAS_NULL): + *numOfNull = pColData->nVal; + break; + case HAS_VALUE: + *numOfNull = 0; + break; + case (HAS_VALUE | HAS_NULL): + case (HAS_VALUE | HAS_NONE): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT1(pColData->pBitMap, iVal) == 0) { + (*numOfNull)++; + } + } + break; + case (HAS_VALUE | HAS_NONE | HAS_NULL): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT2(pColData->pBitMap, iVal) != 2) { + (*numOfNull)++; + } + } + break; + default: + ASSERT(0); + break; + } +} + void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = { NULL, tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL @@ -3530,14 +3568,14 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_ tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARCHAR tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_NCHAR tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON NULL, // TSDB_DATA_TYPE_VARBINARY NULL, // TSDB_DATA_TYPE_DECIMAL NULL, // TSDB_DATA_TYPE_BLOB diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 50fd9d7aa7..b25ab393da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -523,7 +523,7 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type) || ((pColData->flag & HAS_VALUE) == 0)) continue; + if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue; SColumnDataAgg sma = {.colId = pColData->cid}; tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); From 7359719429b5403bbfd3c28a645fad72b534b356 Mon Sep 17 00:00:00 2001 From: Li Ya Qiang Date: Wed, 24 May 2023 13:57:59 +0800 Subject: [PATCH 077/106] implement TD-24378 --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 748b79bd7f..9c141a4f37 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -428,7 +428,7 @@ if [ "$exitcode" != "0" ]; then exit $exitcode fi -if [ -n "${taostools_bin_files}" ]; then +if [ -n "${taostools_bin_files}" ] && [ "$verMode" != "cloud" ]; then wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh" if [ "$osType" != "Darwin" ]; then tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || : From 89bea7100f5cf92f7aa332b167c6d6a850ddd539 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 24 May 2023 00:31:05 -0700 Subject: [PATCH 078/106] fix: fix count var type error --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 +++++++++++--- source/libs/executor/src/scanoperator.c | 5 +++-- source/libs/function/src/builtinsimpl.c | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cb83a561d2..5a7bebebff 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -184,7 +184,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pT SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); void tsdbReaderClose(STsdbReader *pReader); int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4acc784aee..5d970ce6c3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4973,7 +4973,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { return code; } -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { +static bool doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { + bool hasNullSMA = false; // do fill all null column value SMA info int32_t i = 0, j = 0; int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); @@ -4993,6 +4994,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; size++; + hasNullSMA = true; } j += 1; } @@ -5003,12 +5005,15 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; + hasNullSMA = true; } j++; } + + return hasNullSMA; } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool *hasNullSMA) { SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; int32_t code = 0; @@ -5072,7 +5077,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } // do fill all null column value SMA info - doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg); + if (doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg)) { + *hasNullSMA = true; + return TSDB_CODE_SUCCESS; + } size_t size = taosArrayGetSize(pSup->pColAgg); int32_t i = 0, j = 0; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a001d99408..c1430c4ce0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -228,12 +228,13 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { bool allColumnsHaveAgg = true; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + bool hasNullSMA = false; + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg, &hasNullSMA); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - if (!allColumnsHaveAgg) { + if (!allColumnsHaveAgg || hasNullSMA) { return false; } return true; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4c019b3e71..cdecc8b8cc 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -500,7 +500,7 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) { */ SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) { + if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; } else { if (pInputCol->hasNull) { From bd45785dfdd4e88b815069eabd8c6eeebd6f3c78 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 24 May 2023 01:30:48 -0700 Subject: [PATCH 079/106] add test cases --- tests/system-test/2-query/count_null.py | 144 ++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 tests/system-test/2-query/count_null.py diff --git a/tests/system-test/2-query/count_null.py b/tests/system-test/2-query/count_null.py new file mode 100644 index 0000000000..6d2c8db8d6 --- /dev/null +++ b/tests/system-test/2-query/count_null.py @@ -0,0 +1,144 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def check_results(self): + tdSql.query(f"select count(*) from tb1") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c2) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c3) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c4) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c5) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c6) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c7) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb1") + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select count(*) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c2) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c3) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c4) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c5) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c6) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c7) from tb2") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb2") + tdSql.checkData(0, 0, 0) + + for i in range (3, 6): + tdSql.query(f"select count(*) from tb{i}") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c2) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c3) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c4) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c5) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c6) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c7) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c8) from tb{i}") + tdSql.checkData(0, 0, 10000) + + + def run(self): + dbname = 'db' + tbnames = ['tb1', 'tb2', 'tb3', 'tb4', 'tb5', 'tb6'] + num_rows = 20000 + num_tables = 6 + ts_base = 1685548800000 + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + for i in range (num_tables): + tdSql.execute( + f'''create table if not exists {dbname}.{tbnames[i]} + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) + + ''' + ) + + + tdLog.printNoPrefix("==========step2:insert data") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[0]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[1]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, null, null, null, null, null, null, null, 'binary', 'nchar')") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i < num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i >= num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + + tdLog.printNoPrefix("==========step3:check result in memory") + self.check_results(); + + tdLog.printNoPrefix("==========step3:check result from disk") + tdSql.execute(f"flush database db") + self.check_results(); + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From f2b438622a17cd76d5600289074fa192d52779c0 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 24 May 2023 19:47:56 +0800 Subject: [PATCH 080/106] skip some errors --- tests/pytest/crash_gen/crash_gen_main.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 9228cecdb5..5024f1e2fe 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2043,18 +2043,19 @@ class TdSuperTable: for topic in current_topic_list: topic_list.append(topic) - consumer.subscribe(topic_list) - - # consumer with random work life - time_start = time.time() - while 1: - res = consumer.poll(1) - consumer.commit(res) - if time.time() - time_start > random.randint(5, 50): - break try: + consumer.subscribe(topic_list) + + # consumer with random work life + time_start = time.time() + while 1: + res = consumer.poll(1) + consumer.commit(res) + if time.time() - time_start > random.randint(5, 50): + break consumer.unsubscribe() - except TmqError as e: + consumer.close() + except TmqError as err: # topic deleted by other threads pass return From 459c1e6c0cbafc55bc05f34846442b14bc891bb1 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 25 May 2023 13:30:43 +0800 Subject: [PATCH 081/106] fix: fix timestamp conversion error due to daylight saving time --- source/common/src/ttime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index dbb31374de..0aeb420b67 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -93,13 +93,13 @@ int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 'T'); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 'T'); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 'T'); } } else { if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, utime, timePrec, 0); } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 0); + return parseLocaltimeDst((char*)timestr, len, utime, timePrec, 0); } } } From a57da3bdf791e0d9912d4b84dfeaa66595c88f0c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 25 May 2023 14:13:34 +0800 Subject: [PATCH 082/106] fix: fix log not compressed when logKeepDays set to positive value --- source/util/src/tlog.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e3b1c2fa2f..c405fb6a25 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -210,15 +210,15 @@ static void taosKeepOldLog(char *oldName) { (void)taosRenameFile(oldName, fileName); - if (tsLogKeepDays < 0) { - char compressFileName[LOG_FILE_NAME_LEN + 20]; - snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); - if (taosCompressFile(fileName, compressFileName) == 0) { - (void)taosRemoveFile(fileName); - } + char compressFileName[LOG_FILE_NAME_LEN + 20]; + snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); + if (taosCompressFile(fileName, compressFileName) == 0) { + (void)taosRemoveFile(fileName); } - taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays)); + if (tsLogKeepDays > 0) { + taosRemoveOldFiles(tsLogDir, tsLogKeepDays); + } } static void *taosThreadToOpenNewFile(void *param) { From abca0495b113f901f19d848e110f646c3c0c7607 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Thu, 25 May 2023 18:54:22 +0800 Subject: [PATCH 083/106] enh(taosAdapter): change taosAdapter to main branch --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index c67918351d..13826a1a74 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG 565ca21 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 409656f7b6c81aa61aee935ba403825135565c9d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 10:05:54 +0800 Subject: [PATCH 084/106] fix(query): fix bug in iterating the table list. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5d970ce6c3..2eff4ff8da 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3433,6 +3433,8 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { if (!hasNexTable) { return TSDB_CODE_SUCCESS; } + + pBlockScanInfo = pStatus->pTableIter; } initMemDataIterator(*pBlockScanInfo, pReader); From 74f753606b69019d5cd9d415551c3294bb51dfde Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 26 May 2023 10:37:36 +0800 Subject: [PATCH 085/106] fix(query): add regression test case. --- tests/script/tsim/compute/last_row.sim | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/script/tsim/compute/last_row.sim b/tests/script/tsim/compute/last_row.sim index 2e060dc285..8e62fbffb5 100644 --- a/tests/script/tsim/compute/last_row.sim +++ b/tests/script/tsim/compute/last_row.sim @@ -213,4 +213,54 @@ if $rows != 2 then return -1 endi +print =======================> regresss bug in last_row query +sql drop database if exists db; +sql create database if not exists db vgroups 1 cachemodel 'both'; +sql create table db.stb (ts timestamp, c0 bigint) tags(t1 int); +sql insert into db.stb_0 using db.stb tags(1) values ('2023-11-23 19:06:40.000', 491173569); +sql insert into db.stb_2 using db.stb tags(3) values ('2023-11-25 19:30:00.000', 2080726142); +sql insert into db.stb_3 using db.stb tags(4) values ('2023-11-26 06:48:20.000', 1907405128); +sql insert into db.stb_4 using db.stb tags(5) values ('2023-11-24 22:56:40.000', 220783803); + +sql create table db.stb_1 using db.stb tags(2); +sql insert into db.stb_1 (ts) values('2023-11-26 13:11:40.000'); +sql insert into db.stb_1 (ts, c0) values('2023-11-26 13:11:39.000', 11); + +sql select tbname,ts,last_row(c0) from db.stb; +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data01 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +sql alter database db cachemodel 'none'; +sql reset query cache; +sql select tbname,last_row(c0, ts) from db.stb; + +if $rows != 1 then + return -1 +endi + +if $data00 != @stb_1@ then + return -1 +endi + +if $data02 != @23-11-26 13:11:40.000@ then + return -1 +endi + +if $data01 != NULL then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From faaa9da88748d7b13d645bc2ca3ec42e92bd4359 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 26 May 2023 18:17:05 +0800 Subject: [PATCH 086/106] fix: sliding can't be 1a issue --- source/common/src/tglobal.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index bef63d8a49..2a9c54c189 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -139,8 +139,8 @@ int32_t tsCompressColData = -1; // count/hyperloglog function always return values in case of all NULL data or Empty data set. int32_t tsCountAlwaysReturnValue = 1; -// 10 ms for sliding time, the value will changed in case of time precision changed -int32_t tsMinSlidingTime = 10; +// 1 ms for sliding time, the value will changed in case of time precision changed +int32_t tsMinSlidingTime = 1; // the maxinum number of distict query result int32_t tsMaxNumOfDistinctResults = 1000 * 10000; @@ -412,7 +412,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; From 50487255b33a79f2d63f6009db00426d8e799c88 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 26 May 2023 19:02:40 +0800 Subject: [PATCH 087/106] fix:put poll to push manager if wal not exist when offset is latest --- source/dnode/vnode/src/inc/tq.h | 1 - source/dnode/vnode/src/tq/tqUtil.c | 61 ++++++----------------------- source/libs/executor/src/executor.c | 3 ++ 3 files changed, 16 insertions(+), 49 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index ef36b8429a..e6ba5d8636 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -106,7 +106,6 @@ typedef struct { // STqPushHandle pushHandle; // push STqExecHandle execHandle; // exec SRpcMsg* msg; - int32_t noDataPollCnt; tq_handle_status status; } STqHandle; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 622cc5b049..33d2a238c5 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -16,7 +16,6 @@ #include "tq.h" #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) -#define NO_POLL_CNT 5 static int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp); @@ -88,15 +87,12 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { return 0; } -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, bool* pBlockReturned) { +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest) { uint64_t consumerId = pRequest->consumerId; STqOffsetVal reqOffset = pRequest->reqOffset; STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); int32_t vgId = TD_VID(pTq->pVnode); - *pBlockReturned = false; - // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. if (pOffset != NULL) { *pOffsetVal = pOffset->val; @@ -129,28 +125,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); - - tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.rspOffset.version); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); - tDeleteSMqDataRsp(&dataRsp); - - *pBlockReturned = true; - return code; - } else { - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); - tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - - *pBlockReturned = true; - return code; - } + // offset set to previous version when init + tqOffsetResetToLog(pOffsetVal, walGetLastVer(pTq->pVnode->pWal)); } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); @@ -173,25 +149,19 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); code = tqScanData(pTq, pHandle, &dataRsp, pOffset); - if(code != 0) { + if(code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { goto end; } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { - if(pHandle->noDataPollCnt >= NO_POLL_CNT){ // send poll result to client if no data 5 times to avoid lost data - pHandle->noDataPollCnt = 0; - // lock - taosWLockLatch(&pTq->lock); - code = tqRegisterPushHandle(pTq, pHandle, pMsg); - taosWUnLockLatch(&pTq->lock); - tDeleteSMqDataRsp(&dataRsp); - return code; - } - else{ - pHandle->noDataPollCnt++; - } + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST || (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && + dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId)) { + // lock + taosWLockLatch(&pTq->lock); + code = tqRegisterPushHandle(pTq, pHandle, pMsg); + taosWUnLockLatch(&pTq->lock); + tDeleteSMqDataRsp(&dataRsp); + return code; } // NOTE: this pHandle->consumerId may have been changed already. @@ -326,16 +296,11 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ // 1. reset the offset if needed if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { // handle the reset offset cases, according to the consumer's choice. - bool blockReturned = false; - code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest); if (code != 0) { return code; } - // empty block returned, quit - if (blockReturned) { - return 0; - } } else { // use the consumer specified offset // the offset value can not be monotonious increase?? offset = reqOffset; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a73deffa52..fb0b9957f8 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1080,6 +1080,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume + char buf[80] = {0}; + tFormatOffset(buf, 80, &pTaskInfo->streamInfo.currentOffset); + qDebug("currentOffset:%s", buf); if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } From a3e214b9e8eb24b842ad22b96f09e83bcf9e2632 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 26 May 2023 19:17:04 +0800 Subject: [PATCH 088/106] fix:put poll to push manager if wal not exist when offset is latest --- include/libs/wal/wal.h | 3 ++- source/dnode/vnode/src/tq/tqUtil.c | 12 +++--------- source/libs/executor/src/executor.c | 3 --- source/libs/wal/src/walRef.c | 17 +++++++++-------- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 7e106eefde..7af218b78e 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -205,7 +205,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead); int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead); int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead); -SWalRef *walRefFirstVer(SWal *, SWalRef *); +void walRefFirstVer(SWal *, SWalRef *); +void walRefLastVer(SWal *, SWalRef *); SWalRef *walRefCommittedVer(SWal *); SWalRef *walOpenRef(SWal *); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 33d2a238c5..5eaf7b240b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -115,18 +115,12 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand tqOffsetResetToData(pOffsetVal, 0, 0); } } else { - pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); - if (pHandle->pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - // offset set to previous version when init + walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - // offset set to previous version when init - tqOffsetResetToLog(pOffsetVal, walGetLastVer(pTq->pVnode->pWal)); + walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); + tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index fb0b9957f8..a73deffa52 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1080,9 +1080,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT const char* id = GET_TASKID(pTaskInfo); // if pOffset equal to current offset, means continue consume - char buf[80] = {0}; - tFormatOffset(buf, 80, &pTaskInfo->streamInfo.currentOffset); - qDebug("currentOffset:%s", buf); if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index 6aba661926..eb36389f1d 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -63,21 +63,22 @@ int32_t walSetRefVer(SWalRef *pRef, int64_t ver) { return 0; } -SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) { - if (pRef == NULL) { - pRef = walOpenRef(pWal); - if (pRef == NULL) { - return NULL; - } - } +void walRefFirstVer(SWal *pWal, SWalRef *pRef) { taosThreadMutexLock(&pWal->mutex); int64_t ver = walGetFirstVer(pWal); pRef->refVer = ver; taosThreadMutexUnlock(&pWal->mutex); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver); +} - return pRef; +void walRefLastVer(SWal *pWal, SWalRef *pRef) { + taosThreadMutexLock(&pWal->mutex); + int64_t ver = walGetLastVer(pWal); + pRef->refVer = ver; + + taosThreadMutexUnlock(&pWal->mutex); + wDebug("vgId:%d, wal ref version %" PRId64 " for last", pWal->cfg.vgId, ver); } SWalRef *walRefCommittedVer(SWal *pWal) { From 6368b27499bebda87ef4059d1b542ee9bb67c2c2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 29 May 2023 08:50:57 +0800 Subject: [PATCH 089/106] fix: sliding case issue --- tests/script/tsim/parser/sliding.sim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/script/tsim/parser/sliding.sim b/tests/script/tsim/parser/sliding.sim index 1cb4cb5340..7aa69ce9a9 100644 --- a/tests/script/tsim/parser/sliding.sim +++ b/tests/script/tsim/parser/sliding.sim @@ -450,10 +450,11 @@ endi print ====================>check boundary check crash at client side sql select count(*) from sliding_mt0 where ts>now and ts < now-1h; +sql select sum(c1) from sliding_tb0 interval(1a) sliding(1a); + print ========================query on super table print ========================error case -sql_error select sum(c1) from sliding_tb0 interval(1a) sliding(1a); sql_error select sum(c1) from sliding_tb0 interval(10a) sliding(12a); sql_error select sum(c1) from sliding_tb0 sliding(1n) interval(1y); sql_error select sum(c1) from sliding_tb0 interval(-1y) sliding(1n); From 43c0656df2fa0ff5f864185eed902cb8a2fa1d52 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 29 May 2023 10:45:26 +0800 Subject: [PATCH 090/106] fix: if block row num is less than definit min rows, set the bucket index to 0 --- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5d970ce6c3..371ea77b4e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -5242,6 +5242,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { } static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) { + if (numOfRows < startRow) { + return 0; + } int32_t bucketIndex = ((numOfRows - startRow) / bucketRange); if (bucketIndex == numOfBucket) { bucketIndex -= 1; From 3bbf5405c6f9d80521c76b065166be765d0ed435 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 29 May 2023 14:36:29 +0800 Subject: [PATCH 091/106] fix(query): invalid buf page --- source/libs/executor/inc/executil.h | 3 +++ source/util/src/tpagedbuf.c | 1 + 2 files changed, 4 insertions(+) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 5b248f9a90..0f169b0c2e 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -135,6 +135,9 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); + if (!bufPage) { + return NULL; + } if (forUpdate) { setBufPageDirty(bufPage, true); } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index fa8b5d33b7..218a47ea62 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -482,6 +482,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); if (*pInfo != *pi) { + terrno = TSDB_CODE_QRY_SYS_ERROR; uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id); return NULL; } From 5eb1c559e5553f3d6bff2ab71bfd7e11e6d4d1da Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 15:51:00 +0800 Subject: [PATCH 092/106] fix:save offset if latest --- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/dnode/vnode/src/tq/tqUtil.c | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 17eac7d096..7557ddb4e9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -377,7 +377,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } taosWUnLockLatch(&pTq->lock); - tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); + tqDebug("tmq poll: consumer:0x%" PRIx64 "vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", consumerId, vgId, req.subKey, pHandle); taosMsleep(10); } @@ -410,7 +410,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { while (tqIsHandleExec(pHandle)) { - tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 5ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); + tqDebug("vgId:%d, topic:%s, subscription is executing, wait for 10ms and retry, pHandle:%p", vgId, pHandle->subKey, pHandle); taosMsleep(10); } if (pHandle->pRef) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5eaf7b240b..e73aed8966 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -121,6 +121,12 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); + STqOffset offset = {0}; + strcpy(offset.subKey, pRequest->subKey); + if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + return -1; + } } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, vgId, pRequest->subKey); From f984e67eaa33dd4f5e2fb8ce06d4a97c80b0696b Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 29 May 2023 16:04:35 +0800 Subject: [PATCH 093/106] fix(query): invalid buffer page --- source/libs/executor/inc/executil.h | 1 + source/util/src/tpagedbuf.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 0f169b0c2e..05fdba61d7 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -136,6 +136,7 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); if (!bufPage) { + uFatal("failed to get the buffer page:%d since %s", pos->pageId, terrstr()); return NULL; } if (forUpdate) { diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 218a47ea62..2fb189ec9d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -482,7 +482,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); if (*pInfo != *pi) { - terrno = TSDB_CODE_QRY_SYS_ERROR; + terrno = TSDB_CODE_APP_ERROR; uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id); return NULL; } From ccf57129bb6cadda1307b528012ab066ba474eeb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 29 May 2023 16:33:31 +0800 Subject: [PATCH 094/106] fix: show create table privilege issue --- source/libs/catalog/src/ctgUtil.c | 4 ++-- source/libs/parser/src/parAstParser.c | 2 +- source/libs/parser/src/parAuthenticator.c | 2 +- source/libs/parser/src/parUtil.c | 11 +++++++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index ae8fe0cca4..ca63f794e7 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1482,7 +1482,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { switch (pReq->type) { case AUTH_TYPE_READ: { - if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { req->singleType = AUTH_TYPE_READ; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { @@ -1498,7 +1498,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { break; } case AUTH_TYPE_WRITE: { - if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { + if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { req->singleType = AUTH_TYPE_WRITE; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); if (pRes->pass || res->metaNotExists) { diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5db1f5dbdb..df080b574c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -581,7 +581,7 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_READ, + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, pCxt->pMetaCache); } return code; diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 1586d8128b..251d3bd0cb 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -175,7 +175,7 @@ static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) { static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) { SNode* pTagCond = NULL; // todo check tag condition for subtable - return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_READ, &pTagCond); + return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, &pTagCond); } static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 14da6f8aab..9c95e2b17c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -498,7 +498,7 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, char* pStr) { - return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL != pTable && '\0' == pTable[0]) ? NULL : pTable, + return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, type); } @@ -524,6 +524,9 @@ static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { strncpy(pStr, pStart, p - pStart); *pNext = ++p; } + if (*pStart == '`' && *(pStart + 1) == '`') { + *pStr = 0; + } } static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) { @@ -532,7 +535,11 @@ static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUser pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p); getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p); getStringFromAuthStr(p, pUserAuth->tbName.tname, &p); - pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + if (pUserAuth->tbName.tname[0]) { + pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + } else { + pUserAuth->tbName.type = TSDB_DB_NAME_T; + } pUserAuth->type = getIntegerFromAuthStr(p, &p); } From 9eb15e41efc8a3b936746151163018420bf17b48 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 29 May 2023 16:44:03 +0800 Subject: [PATCH 095/106] fix:put poll to push manager if wal not exist when offset is latest --- source/dnode/vnode/src/tq/tqUtil.c | 40 ++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index e73aed8966..d30ed638f1 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -87,12 +87,13 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { return 0; } -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest) { +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, bool* pBlockReturned) { uint64_t consumerId = pRequest->consumerId; STqOffsetVal reqOffset = pRequest->reqOffset; STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); int32_t vgId = TD_VID(pTq->pVnode); + *pBlockReturned = false; // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. if (pOffset != NULL) { *pOffsetVal = pOffset->val; @@ -120,12 +121,27 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { walRefLastVer(pTq->pVnode->pWal, pHandle->pRef); - tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer); - STqOffset offset = {0}; - strcpy(offset.subKey, pRequest->subKey); - if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { - terrno = TSDB_CODE_PAR_INTERNAL_ERROR; - return -1; + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); + + tqOffsetResetToLog(&dataRsp.rspOffset, pHandle->pRef->refVer); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, + pHandle->subKey, vgId, dataRsp.rspOffset.version); + int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + tDeleteSMqDataRsp(&dataRsp); + + *pBlockReturned = true; + return code; + } else { + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, pRequest); + tqOffsetResetToLog(&taosxRsp.rspOffset, pHandle->pRef->refVer); + int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); + tDeleteSTaosxRsp(&taosxRsp); + + *pBlockReturned = true; + return code; } } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", @@ -154,8 +170,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST || (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId)) { + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST && dataRsp.blockNum == 0) { // lock taosWLockLatch(&pTq->lock); code = tqRegisterPushHandle(pTq, pHandle, pMsg); @@ -296,11 +311,16 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ // 1. reset the offset if needed if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { // handle the reset offset cases, according to the consumer's choice. - code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest); + bool blockReturned = false; + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); if (code != 0) { return code; } + // empty block returned, quit + if (blockReturned) { + return 0; + } } else { // use the consumer specified offset // the offset value can not be monotonious increase?? offset = reqOffset; From 6a7d8858dc516c3b1fe79ea2194e5159871ac99e Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 14:36:51 +0800 Subject: [PATCH 096/106] test: add smaBasic.py test case --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/smaBasic.py | 297 ++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 tests/system-test/2-query/smaBasic.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcfb392bb4..f0543f2456 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -294,6 +294,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaBasic.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py new file mode 100644 index 0000000000..917a2d3a08 --- /dev/null +++ b/tests/system-test/2-query/smaBasic.py @@ -0,0 +1,297 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +import time + +import taos +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + + # get col value and total max min ... + def getColsValue(self, i, j): + # c1 value + if random.randint(1, 10) == 5: + c1 = None + else: + c1 = 1 + + # c2 value + if j % 3200 == 0: + c2 = 8764231 + elif random.randint(1, 10) == 5: + c2 = None + else: + c2 = random.randint(-87654297, 98765321) + + + value = f"({self.ts}, " + self.ts += 1 + + # c1 + if c1 is None: + value += "null," + else: + self.c1Cnt += 1 + value += f"{c1}," + # c2 + if c2 is None: + value += "null)" + else: + value += f"{c2})" + # total count + self.c2Cnt += 1 + # max + if self.c2Max is None: + self.c2Max = c2 + else: + if c2 > self.c2Max: + self.c2Max = c2 + # min + if self.c2Min is None: + self.c2Min = c2 + else: + if c2 < self.c2Min: + self.c2Min = c2 + # sum + if self.c2Sum is None: + self.c2Sum = c2 + else: + self.c2Sum += c2 + + return value + + # insert data + def insertData(self): + tdLog.info("insert data ....") + sqls = "" + for i in range(self.childCnt): + # insert child table + values = "" + pre_insert = f"insert into t{i} values " + for j in range(self.childRow): + if values == "": + values = self.getColsValue(i, j) + else: + values += "," + self.getColsValue(i, j) + + # batch insert + if j % self.batchSize == 0 and values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + # append last + if values != "": + sql = pre_insert + values + tdSql.execute(sql) + values = "" + + sql = "flush database db;" + tdLog.info(sql) + tdSql.execute(sql) + # insert finished + tdLog.info(f"insert data successfully.\n" + f" inserted child table = {self.childCnt}\n" + f" inserted child rows = {self.childRow}\n" + f" total inserted rows = {self.childCnt*self.childRow}\n") + return + + + # prepareEnv + def prepareEnv(self): + # init + self.ts = 1600000000000 + self.childCnt = 5 + self.childRow = 2000000 + self.batchSize = 2000 + self.c1_values = [] + + # total + self.c1Cnt = 0 + self.c2Cnt = 0 + self.c2Max = None + self.c2Min = None + self.c2Sum = None + + # create database db + sql = f"create database db vgroups 5 replica 3" + tdLog.info(sql) + tdSql.execute(sql) + sql = f"use db" + tdSql.execute(sql) + + # create super talbe st + sql = f"create table st(ts timestamp, c1 int, c2 bigint) tags(area int)" + tdLog.info(sql) + tdSql.execute(sql) + + # create child table + for i in range(self.childCnt): + sql = f"create table t{i} using st tags({i}) " + tdSql.execute(sql) + + # insert data + self.insertData() + + # query sql value + def queryValue(self, sql): + tdSql.query(sql) + return tdSql.getData(0, 0) + + # sum + def checkCorrentSum(self): + # query count + sql = "select sum(c1) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c1Cnt: + tdLog.exit(f"Sum Not Expect. expect={self.c1Cnt} query={val} sql:{sql}") + return + + # not + sql1 = "select sum(c1) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + # over than + sql1 = "select sum(c1) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = "select sum(c1) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + sql3 = "select sum(c1) from st where c2 is null" + val3 = self.queryValue(sql3) + if val != val1 + val2 + val3: + tdLog.exit(f"Sum Not Equal. val != val1 + val2 + val3. val={val} val1={val1} val2={val2} val2={val3} sql1={sql1} sql2={sql2} sql2={sql3}") + return + + tdLog.info(f"check correct sum on c1 successfully.") + + # check result + def checkResult(self, fun, val, val1, val2, sql1, sql2): + if fun == "count": + if val != val1 + val2: + tdLog.exit(f"{fun} NOT SAME. val != val1 + val2. val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "max": + if val != max([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != max(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + elif fun == "min": + if val != min([val1, val2]): + tdLog.exit(f"{fun} NOT SAME . val != min(val1 ,val2) val={val} val1={val1} val2={val2} sql1={sql1} sql2={sql2}") + return + + # sum + def checkCorrentFun(self, fun, expectVal): + # query + sql = f"select {fun}(c2) from st" + val = self.queryValue(sql) + if val != expectVal: + tdLog.exit(f"{fun} Not Expect. expect={expectVal} query={val} sql:{sql}") + return + + # not + sql1 = f"select {fun}(c2) from st where c2 = 8764231" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 != 8764231" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # over than + sql1 = f"select {fun}(c2) from st where c2 > 8000" + val1 = self.queryValue(sql1) + sql2 = f"select {fun}(c2) from st where c2 <= 8000" + val2 = self.queryValue(sql2) + self.checkResult(fun, val, val1, val2, sql1, sql2) + + # successful + tdLog.info(f"check correct {fun} on c2 successfully.") + + # check query corrent + def checkCorrect(self): + # count + self.checkCorrentFun("count", self.c2Cnt) + # max + self.checkCorrentFun("max", self.c2Max) + # min + self.checkCorrentFun("min", self.c2Min) + # sum + self.checkCorrentSum() + + # c2 sum + sql = "select sum(c2) from st" + val = self.queryValue(sql) + # c1Sum is equal c1Cnt + if val != self.c2Sum: + tdLog.exit(f"c2 Sum Not Expect. expect={self.c2Sum} query={val} sql:{sql}") + return + + def checkPerformance(self): + # have sma caculate + sql1 = "select count(*) from st" + stime = time.time() + tdSql.execute(sql1, 1) + spend1 = time.time() - stime + + + # no sma caculate + sql2 = "select count(*) from st where c2 != 8764231 or c2 is null" + stime = time.time() + tdSql.execute(sql2, 1) + spend2 = time.time() - stime + + time1 = "%.2f"%(spend1*1000) + time2 = "%.2f"%(spend2*1000) + if spend2 < spend1 * 5: + tdLog.exit(f"performance not passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + return + tdLog.info(f"performance passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") + + + # init + def init(self, conn, logSql, replicaVar=1): + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + # run + def run(self): + # prepare env + self.prepareEnv() + + # query + self.checkCorrect() + + # performance + self.checkPerformance() + + # stop + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From ed9eaf541d1d95afac64c18898d2f384a0308031 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 14:37:46 +0800 Subject: [PATCH 097/106] test: add smaBasic.py test case --- tests/system-test/2-query/smaBasic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py index 917a2d3a08..26adc03d2b 100644 --- a/tests/system-test/2-query/smaBasic.py +++ b/tests/system-test/2-query/smaBasic.py @@ -117,7 +117,7 @@ class TDTestCase: self.ts = 1600000000000 self.childCnt = 5 self.childRow = 2000000 - self.batchSize = 2000 + self.batchSize = 5000 self.c1_values = [] # total From 772ef3bd19b5264d54766bb7397b7c135d741cde Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 30 May 2023 15:39:15 +0800 Subject: [PATCH 098/106] test: modfiy insert rows to small --- tests/system-test/2-query/smaBasic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/smaBasic.py b/tests/system-test/2-query/smaBasic.py index 26adc03d2b..43c379ee53 100644 --- a/tests/system-test/2-query/smaBasic.py +++ b/tests/system-test/2-query/smaBasic.py @@ -116,9 +116,8 @@ class TDTestCase: # init self.ts = 1600000000000 self.childCnt = 5 - self.childRow = 2000000 + self.childRow = 1000000 self.batchSize = 5000 - self.c1_values = [] # total self.c1Cnt = 0 @@ -262,7 +261,7 @@ class TDTestCase: time1 = "%.2f"%(spend1*1000) time2 = "%.2f"%(spend2*1000) - if spend2 < spend1 * 5: + if spend2 < spend1 * 8: tdLog.exit(f"performance not passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") return tdLog.info(f"performance passed! sma spend1={time1}ms no sma spend2= {time2}ms sql1={sql1} sql2= {sql2}") From 23bcfc2fd0714f9ef549c638635fc26ec7e31f2d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 31 May 2023 09:16:47 +0800 Subject: [PATCH 099/106] enh: stmt column length validation --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 5 +- source/libs/parser/src/parInsertStmt.c | 7 +- tests/script/api/batchprepare.c | 93 +++++++++++++++++++------- 4 files changed, 79 insertions(+), 28 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 8be5cb4d41..e04bdd1b07 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -145,7 +145,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); // for stmt bind -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen); void tColDataSortMerge(SArray *colDataArr); // for raw block diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 55204045ba..688f3b006c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2503,7 +2503,7 @@ _exit: return code; } -int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen) { int32_t code = 0; if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { @@ -2515,6 +2515,9 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { if (pBind->is_null && pBind->is_null[i]) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; + } else if (pBind->length[i] > buffMaxLen) { + uError("var data length too big, len:%d, max:%d", pBind->length[i], buffMaxLen); + return TSDB_CODE_INVALID_PARA; } else { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 922a0f45ff..8284913975 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -266,7 +266,10 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in pBind = bind + c; } - tColDataAddValueByBind(pCol, pBind); + code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); + if (code) { + goto _return; + } } qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); @@ -309,7 +312,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu pBind = bind; } - tColDataAddValueByBind(pCol, pBind); + tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); qDebug("stmt col %d bind %d rows data", colIdx, rowNum); diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 99507ef5c3..80bf5b90af 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -16,8 +16,8 @@ int32_t shortColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; int32_t fullColList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_UINT, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BINARY, TSDB_DATA_TYPE_NCHAR}; -int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT}; -int32_t optrIdxList[] = {0, 7}; +int32_t bindColTypeList[] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_NCHAR}; +int32_t optrIdxList[] = {5, 11}; typedef struct { char* oper; @@ -123,6 +123,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos); +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos); enum { TTYPE_INSERT = 1, @@ -190,6 +191,7 @@ CaseCfg gCase[] = { {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:NG-TBNEXISTS",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, false, insertNonExistsTb, 10, 10, 1, 3, 0, 0, 1, -1}, + {"query:NG-VARLENERR",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, true, insertVarLenErr, 10, 10, 1, 3, 0, 0, 1, -1}, // {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, // {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, @@ -319,7 +321,7 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper #if 0 CaseCtrl gCaseCtrl = { // query case with specified col&oper - .bindNullNum = 1, + .bindNullNum = 0, .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, @@ -329,18 +331,19 @@ CaseCtrl gCaseCtrl = { // query case with specified col&oper .bindTagNum = 0, .bindRowNum = 0, .bindColTypeNum = 0, - .bindColTypeList = NULL, + .bindColTypeList = bindColTypeList, .optrIdxListNum = 0, - .optrIdxList = NULL, + .optrIdxList = optrIdxList, .checkParamNum = false, .printRes = true, .runTimes = 0, .caseRunIdx = -1, - //.optrIdxListNum = tListLen(optrIdxList), - //.optrIdxList = optrIdxList, - //.bindColTypeNum = tListLen(bindColTypeList), - //.bindColTypeList = bindColTypeList, - .caseIdx = 8, + .optrIdxListNum = tListLen(optrIdxList), + .optrIdxList = optrIdxList, + .bindColTypeNum = tListLen(bindColTypeList), + .bindColTypeList = bindColTypeList, + .caseRunIdx = -1, + .caseIdx = 24, .caseNum = 1, .caseRunNum = 1, }; @@ -1439,14 +1442,17 @@ void bpShowBindParam(TAOS_MULTI_BIND *bind, int32_t num) { } } -int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { +int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, bool expectFail) { static int32_t n = 0; - bpCheckColFields(stmt, bind); + if (!expectFail) { + bpCheckColFields(stmt, bind); + } if (gCurCase->bindRowNum > 1) { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1454,6 +1460,7 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { if (taos_stmt_bind_single_param_batch(stmt, bind+i, i)) { + if (expectFail) continue; printf("!!!taos_stmt_bind_single_param_batch %d error:%s\n", taos_stmt_errstr(stmt), i); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1463,12 +1470,14 @@ int32_t bpBindParam(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) { } else { if (0 == (n++%2)) { if (taos_stmt_bind_param_batch(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param_batch error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); } } else { if (taos_stmt_bind_param(stmt, bind)) { + if (expectFail) return 0; printf("!!!taos_stmt_bind_param error:%s\n", taos_stmt_errstr(stmt)); bpShowBindParam(bind, gCurCase->bindColNum); exit(1); @@ -1531,7 +1540,7 @@ int insertMBSETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1583,7 +1592,7 @@ int insertMBSETest2(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1641,7 +1650,7 @@ int insertMBMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1691,7 +1700,7 @@ int insertMBMETest2(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1759,7 +1768,7 @@ int insertMBMETest3(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1811,7 +1820,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos) { } } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1872,7 +1881,7 @@ int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -1938,7 +1947,7 @@ int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos) { } for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2005,7 +2014,7 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos) { if (gCaseCtrl.checkParamNum) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2065,7 +2074,7 @@ int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + t*bindTimes*gCurCase->bindColNum + b*gCurCase->bindColNum, false)) { exit(1); } @@ -2119,7 +2128,7 @@ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2167,7 +2176,7 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { bpCheckParamNum(stmt); } - if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum)) { + if (bpBindParam(stmt, data.pBind + n * gCurCase->bindColNum, false)) { exit(1); } @@ -2234,6 +2243,42 @@ int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) { return 0; } +void bpAddWrongVarBuffLen(TAOS_MULTI_BIND* pBind) { + for (int32_t i = 0; i < gCurCase->bindColNum; ++i) { + if (pBind[i].buffer_type == TSDB_DATA_TYPE_BINARY || pBind[i].buffer_type == TSDB_DATA_TYPE_NCHAR) { + *pBind[i].length += 100; + } + } +} + +int insertVarLenErr(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + code = bpSetTableNameTags(&data, 0, "t0", stmt); + if (code != 0){ + printf("!!!taos_stmt_set_tbname error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpAddWrongVarBuffLen(data.pBind); + + if (bpBindParam(stmt, data.pBind, true)) { + exit(1); + } + + destroyData(&data); + + return 0; +} int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { From 6517ede04461b5f471c1859fbbd7d3963e0010df Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 1 Jun 2023 11:23:31 +0800 Subject: [PATCH 100/106] fix: fix interval operator ensure capacity logic --- source/libs/executor/src/executorInt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index e18906ad18..5872d019fa 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -827,7 +827,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - uint32_t newSize = pBlock->info.rows + pRow->numOfRows + (numOfRows - i) > 1 ? 1 : 0; + uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - i) > 1 ? 1 : 0); blockDataEnsureCapacity(pBlock, newSize); qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", newSize, pBlock->info.capacity, GET_TASKID(pTaskInfo)); From 706e35021733713389bdb065d064189e4fb89502 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jun 2023 14:36:42 +0800 Subject: [PATCH 101/106] fix(query): fix error in fill and partition. --- source/libs/executor/inc/tfill.h | 2 ++ source/libs/executor/src/filloperator.c | 28 +++++++++++++++++++------ source/libs/executor/src/tfill.c | 19 ++++++++++++++--- tests/script/tsim/query/partitionby.sim | 4 ++-- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 78b3cd2f40..79837480d7 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -120,6 +120,8 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts); +bool taosFillNotStarted(const SFillInfo* pFillInfo); SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, int32_t numOfNotFillCols, const struct SNodeListNode* val); bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 839c845bac..3d1df6482e 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -139,6 +139,12 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { } SOperatorInfo* pDownstream = pOperator->pDownstream[0]; + + // the scan order may be different from the output result order for agg interval operator. + if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL) { + order = ((SIntervalAggOperatorInfo*) pDownstream->info)->resultTsOrder; + } + while (1) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { @@ -162,11 +168,24 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block pInfo->totalInputRows += pInfo->pRes->info.rows; - if (order == pInfo->pFillInfo->order) { + if (order == TSDB_ORDER_ASC) { + int64_t skey = pBlock->info.window.skey; + if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); + } + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); } else { + int64_t ekey = pBlock->info.window.ekey; + if (ekey > pInfo->pFillInfo->start) { + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); + } + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); } + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block pInfo->existNewGroupBlock = pBlock; @@ -256,11 +275,8 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t const char* id, SInterval* pInterval, int32_t fillType, int32_t order) { SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); - STimeWindow w = {0}; - int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; - - getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); - pInfo->pFillInfo = taosCreateFillInfo(w.skey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, + int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; + pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); if (order == TSDB_ORDER_ASC) { diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index e4d5d2c268..7cc50a70ab 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -517,11 +517,16 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) return; } + // the endKey is now the aligned time window value. truncate time window isn't correct. pFillInfo->end = endKey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval); - pFillInfo->end = taosTimeAdd(pFillInfo->end, pFillInfo->interval.interval, pFillInfo->interval.intervalUnit,pFillInfo->interval.precision); + +#if 0 + if (pFillInfo->order == TSDB_ORDER_ASC) { + ASSERT(pFillInfo->start <= pFillInfo->end); + } else { + ASSERT(pFillInfo->start >= pFillInfo->end); } +#endif pFillInfo->index = 0; pFillInfo->numOfRows = numOfRows; @@ -531,6 +536,13 @@ void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) pFillInfo->pSrcBlock = (SSDataBlock*)pInput; } +void taosFillUpdateStartTimestampInfo(SFillInfo* pFillInfo, int64_t ts) { + pFillInfo->start = ts; + pFillInfo->currentKey = ts; +} + +bool taosFillNotStarted(const SFillInfo* pFillInfo) {return pFillInfo->start == pFillInfo->currentKey;} + bool taosFillHasMoreResults(SFillInfo* pFillInfo) { int32_t remain = taosNumOfRemainRows(pFillInfo); if (remain > 0) { @@ -565,6 +577,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } + numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); numOfRes += 1; diff --git a/tests/script/tsim/query/partitionby.sim b/tests/script/tsim/query/partitionby.sim index da7dfc0090..76d4f87908 100644 --- a/tests/script/tsim/query/partitionby.sim +++ b/tests/script/tsim/query/partitionby.sim @@ -121,7 +121,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 2 then +if $data00 != 4 then return -1 endi @@ -129,7 +129,7 @@ sql select count(*) from (select ts from $mt1 where ts is not null partition by if $rows != 1 then return -1 endi -if $data00 != 8 then +if $data00 != 16 then return -1 endi From 1036259a7b707a48902b02cb56a563c432ca37d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jun 2023 18:25:05 +0800 Subject: [PATCH 102/106] fix(query): fix error in fill. --- source/common/src/ttime.c | 1 + source/libs/executor/src/filloperator.c | 87 ++++++++++++++++++------- tests/script/tsim/parser/function.sim | 3 + 3 files changed, 69 insertions(+), 22 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 0aeb420b67..d8c43747f7 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -738,6 +738,7 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char ekey = skey; skey = tmp; } + if (unit != 'n' && unit != 'y') { return (int32_t)((ekey - skey) / interval); } diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 3d1df6482e..e257879a04 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -61,6 +61,7 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); @@ -72,14 +73,16 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; getTableScanInfo(pOperator, &order, &scanFlag, false); - - int64_t ekey = pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); blockDataCleanup(pInfo->pRes); doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ekey); + revisedFillStartKey(pInfo, pInfo->existNewGroupBlock); + + int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows; @@ -119,6 +122,55 @@ void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int pInfo->pRes->info.id.groupId = pBlock->info.id.groupId; } +// todo refactor: decide the start key according to the query time range. +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { + int32_t order = pInfo->pFillInfo->order; + + if (order == TSDB_ORDER_ASC) { + int64_t skey = pBlock->info.window.skey; + if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); + } else if (pInfo->pFillInfo->start < skey) { + int64_t t = skey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev < pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } else { + int64_t ekey = pBlock->info.window.ekey; + if (ekey > pInfo->pFillInfo->start) { + ASSERT( taosFillNotStarted(pInfo->pFillInfo)); + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); + } else if (ekey < pInfo->pFillInfo->start) { + int64_t t = ekey; + SInterval* pInterval = &pInfo->pFillInfo->interval; + + while(1) { + int64_t prev = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + if (prev < pInfo->pFillInfo->start) { + t = prev; + break; + } + t = prev; + } + + // todo time window chosen problem: t or prev value? + taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, t); + } + } +} + static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { SFillOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -164,28 +216,16 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); - if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) { + if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { + if (pInfo->curGroupId == 0) { + revisedFillStartKey(pInfo, pBlock); + } + pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block pInfo->totalInputRows += pInfo->pRes->info.rows; - if (order == TSDB_ORDER_ASC) { - int64_t skey = pBlock->info.window.skey; - if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the - ASSERT( taosFillNotStarted(pInfo->pFillInfo)); - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey); - } - - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.ekey); - } else { - int64_t ekey = pBlock->info.window.ekey; - if (ekey > pInfo->pFillInfo->start) { - ASSERT( taosFillNotStarted(pInfo->pFillInfo)); - taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, ekey); - } - - taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey); - } - + int64_t ts = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey; + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); } else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block pInfo->existNewGroupBlock = pBlock; @@ -276,6 +316,9 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pNotFillExpr, numOfNotFillCols, pValNode); int64_t startKey = (order == TSDB_ORDER_ASC) ? win.skey : win.ekey; + +// STimeWindow w = {0}; +// getInitialStartTimeWindow(pInterval, startKey, &w, order == TSDB_ORDER_ASC); pInfo->pFillInfo = taosCreateFillInfo(startKey, numOfCols, numOfNotFillCols, capacity, pInterval, fillType, pColInfo, pInfo->primaryTsCol, order, id); diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 7f69aa2d02..120c4b8148 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -954,11 +954,14 @@ endi print =========================>TD-5190 sql select _wstart, stddev(f1) from st1 where ts>'2021-07-01 1:1:1' and ts<'2021-07-30 00:00:00' interval(1d) fill(NULL); if $rows != 29 then + print expect 29, actual: $rows return -1 endi + if $data00 != @21-07-01 00:00:00.000@ then return -1 endi + if $data01 != NULL then return -1 endi From 403617976c500b77066519e752e6dffbff7b8e48 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 1 Jun 2023 22:45:03 +0800 Subject: [PATCH 103/106] fix(query): fix error in fill. --- source/libs/executor/src/filloperator.c | 40 ++++++++++++------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index e257879a04..75bfe7b671 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -61,24 +61,24 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; -static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock); +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; SSDataBlock* pResBlock = pInfo->pFinalRes; - int32_t order = TSDB_ORDER_ASC; +// int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - getTableScanInfo(pOperator, &order, &scanFlag, false); +// getTableScanInfo(pOperator, &order, &scanFlag, false); taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); blockDataCleanup(pInfo->pRes); doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); - revisedFillStartKey(pInfo, pInfo->existNewGroupBlock); + revisedFillStartKey(pInfo, pInfo->existNewGroupBlock, order); int64_t ts = (order == TSDB_ORDER_ASC)? pInfo->existNewGroupBlock->info.window.ekey:pInfo->existNewGroupBlock->info.window.skey; taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, ts); @@ -93,7 +93,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp } static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, - SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { + SResultInfo* pResultInfo, int32_t order) { if (taosFillHasMoreResults(pInfo->pFillInfo)) { int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows; taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows); @@ -103,7 +103,7 @@ static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOpera // handle the cached new group data block if (pInfo->existNewGroupBlock) { - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); } } @@ -123,9 +123,7 @@ void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int } // todo refactor: decide the start key according to the query time range. -static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { - int32_t order = pInfo->pFillInfo->order; - +static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order) { if (order == TSDB_ORDER_ASC) { int64_t skey = pBlock->info.window.skey; if (skey < pInfo->pFillInfo->start) { // the start key may be smaller than the @@ -137,7 +135,7 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { while(1) { int64_t prev = taosTimeAdd(t, -pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - if (prev < pInfo->pFillInfo->start) { + if (prev <= pInfo->pFillInfo->start) { t = prev; break; } @@ -158,7 +156,7 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock) { while(1) { int64_t prev = taosTimeAdd(t, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); - if (prev < pInfo->pFillInfo->start) { + if (prev >= pInfo->pFillInfo->start) { t = prev; break; } @@ -184,12 +182,6 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { int32_t scanFlag = MAIN_SCAN; getTableScanInfo(pOperator, &order, &scanFlag, false); - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); - if (pResBlock->info.rows > 0) { - pResBlock->info.id.groupId = pInfo->curGroupId; - return pResBlock; - } - SOperatorInfo* pDownstream = pOperator->pDownstream[0]; // the scan order may be different from the output result order for agg interval operator. @@ -197,6 +189,12 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { order = ((SIntervalAggOperatorInfo*) pDownstream->info)->resultTsOrder; } + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); + if (pResBlock->info.rows > 0) { + pResBlock->info.id.groupId = pInfo->curGroupId; + return pResBlock; + } + while (1) { SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream); if (pBlock == NULL) { @@ -218,7 +216,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { if (pInfo->curGroupId == 0) { - revisedFillStartKey(pInfo, pBlock); + revisedFillStartKey(pInfo, pBlock, order); } pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block @@ -249,7 +247,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { return pResBlock; } - doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; @@ -257,7 +255,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { } else if (pInfo->existNewGroupBlock) { // try next group blockDataCleanup(pResBlock); - doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo); + doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, order); if (pResBlock->info.rows > pResultInfo->threshold) { pResBlock->info.id.groupId = pInfo->curGroupId; return pResBlock; From bbf3f1ccd92af410c0f50bcb6487693cb862ea38 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 2 Jun 2023 00:22:35 +0800 Subject: [PATCH 104/106] fix(query): add more check. --- source/libs/executor/src/filloperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 75bfe7b671..e2862b19c7 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -215,7 +215,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { - if (pInfo->curGroupId == 0) { + if (pInfo->curGroupId == 0 && taosFillNotStarted(pInfo->pFillInfo)) { revisedFillStartKey(pInfo, pBlock, order); } From 274771d7b4cb5329ba1267633e99d527b5a42846 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 2 Jun 2023 08:59:00 +0800 Subject: [PATCH 105/106] TD-24565 --- source/libs/executor/src/scanoperator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 28abee4342..39907be6d2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1072,6 +1072,7 @@ static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t grou void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint64_t ver) { pTableScanInfo->base.cond.twindows = *pWin; + pTableScanInfo->base.cond.startVersion = 0; pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; @@ -1194,6 +1195,7 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ } STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; + qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, pInfo->pUpdateInfo->maxDataVersion); resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); pInfo->pTableScanOp->status = OP_OPENED; return true; @@ -1889,6 +1891,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { TSKEY maxTs = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } else { + pInfo->pUpdateInfo->maxDataVersion = pTaskInfo->streamInfo.fillHistoryVer2; doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); } } From 27f791ac7fc071bc7899d0a0188137686529eaf0 Mon Sep 17 00:00:00 2001 From: liuyao <54liuyao@163.com> Date: Fri, 2 Jun 2023 11:04:02 +0800 Subject: [PATCH 106/106] reset ud table name --- source/dnode/vnode/src/tq/tqSink.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6e02d5e21b..db1b5ed902 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -294,6 +294,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d char* ctbName = pDataBlock->info.parTbName; if (!ctbName[0]) { + memset(ctbName, 0, TSDB_TABLE_NAME_LEN); if (res == TSDB_CODE_SUCCESS) { memcpy(ctbName, pTableSinkInfo->tbName, strlen(pTableSinkInfo->tbName)); } else {