From d17d51194f5d8276f2d013733f4ebdd279e02ffd Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 6 Dec 2023 18:44:55 +0800 Subject: [PATCH 01/27] taos_fetch_row cannot be called before the query callback ends. --- source/client/inc/clientInt.h | 1 + source/client/src/clientImpl.c | 2 ++ source/client/src/clientMain.c | 6 ++++++ tests/taosc_test/taoscTest.cpp | 10 ++++++---- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b4ee619332..a6d5039be7 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -273,6 +273,7 @@ typedef struct SRequestObj { bool killed; bool inRetry; bool isSubReq; + bool inCallback; uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog uint32_t retry; int64_t allocatorRefId; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f615111b7f..148cca8dbc 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2607,7 +2607,9 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param } void doRequestCallback(SRequestObj* pRequest, int32_t code) { + pRequest->inCallback = true; pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code); + pRequest->inCallback = false; } int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1ce7c02dcf..fb36a658e4 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -418,6 +418,12 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } + if(pRequest->inCallback) { + tscError("can not call taos_fetch_row before query callback ends."); + terrno = TSDB_CODE_TSC_INVALID_OPERATION; + return NULL; + } + return doAsyncFetchRows(pRequest, true, true); } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SMqRspObj *msg = ((SMqRspObj *)res); diff --git a/tests/taosc_test/taoscTest.cpp b/tests/taosc_test/taoscTest.cpp index fbdb152ab4..3f49b11b70 100644 --- a/tests/taosc_test/taoscTest.cpp +++ b/tests/taosc_test/taoscTest.cpp @@ -190,10 +190,12 @@ void queryCallback2(void* param, void* res, int32_t code) { ASSERT_TRUE(param == pUserParam); // After using taos_query_a to query, using taos_fetch_row in the callback will cause blocking. // Reason: schProcessOnCbBegin SCH_LOCK_TASK(pTask) - /* TAOS_ROW row; - while ((row = taos_fetch_row(res))) { - getRecordCounts++; - } */ + TAOS_ROW row; + row = taos_fetch_row(res); + ASSERT_TRUE(row == NULL); + int* errCode = taosGetErrno(); + ASSERT_TRUE(*errCode = TSDB_CODE_TSC_INVALID_OPERATION); + tsem_post(&query_sem); taos_free_result(res); } From f2d89b70868b3f9af79515c45adecb24b6b93852 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 15 Dec 2023 11:29:25 +0800 Subject: [PATCH 02/27] fix: use req after release --- source/client/src/clientImpl.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 148cca8dbc..f2111eb2a2 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2608,8 +2608,13 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param void doRequestCallback(SRequestObj* pRequest, int32_t code) { pRequest->inCallback = true; + int64_t this = pRequest->self; pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code); - pRequest->inCallback = false; + SRequestObj* pReq = acquireRequest(this); + if (pReq != NULL) { + pReq->inCallback = false; + releaseRequest(this); + } } int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) { From 74793e652834a9eb005a2de010d08e7a3e7b690d Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Sun, 17 Dec 2023 10:05:58 +0800 Subject: [PATCH 03/27] fix: inCallback --- source/client/src/clientEnv.c | 1 + source/client/src/clientImpl.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 77cda347a4..b6c5701915 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -336,6 +336,7 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) { pRequest->pDb = getDbOfConnection(pTscObj); pRequest->pTscObj = pTscObj; + pRequest->inCallback = false; pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index f2111eb2a2..28a29c2138 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1711,6 +1711,7 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU taos_fetch_rows_a(pRequest, syncFetchFn, &sem); tsem_wait(&sem); tsem_destroy(&sem); + pRequest->inCallback = false; } if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) { @@ -2490,6 +2491,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { if (param->pRequest != NULL) { param->pRequest->syncQuery = true; pRequest = param->pRequest; + param->pRequest->inCallback = false; } taosMemoryFree(param); From 7f8479f9531266fe09fa64a96048ac161e0c22bd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Dec 2023 15:16:33 +0800 Subject: [PATCH 04/27] opti:logic in parse raw line & make log more specific --- source/client/inc/clientSml.h | 1 - source/client/src/clientSml.c | 69 ++++++++++++++++++--------- source/libs/parser/src/parInsertSml.c | 10 +++- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index 9ae28dd55e..b732abffb1 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -171,7 +171,6 @@ typedef struct { int8_t precision; bool reRun; bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; int32_t ttl; int32_t uid; // used for automatic create child table diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 67b23792ad..c392308195 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1640,6 +1640,37 @@ int32_t smlClearForRerun(SSmlHandle *info) { return TSDB_CODE_SUCCESS; } +static bool getLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, + int numLines, int i, char** tmp, int *len){ + if (lines) { + *tmp = lines[i]; + *len = strlen(*tmp); + } else if (rawLine) { + *tmp = rawLine; + while (rawLine < rawLineEnd) { + if (*(rawLine++) == '\n') { + break; + } + (*len)++; + } + if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment + false; + } + } + + if(rawLine != NULL){ + char* print = taosMemoryCalloc(*len + 1, 1); + memcpy(print, *tmp, *len); + uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, + numLines, info->protocol, *len, print); + taosMemoryFree(print); + }else{ + uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id, + numLines, info->protocol, *len, *tmp); + } + return true; +} + static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); int32_t code = TSDB_CODE_SUCCESS; @@ -1661,25 +1692,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char while (i < numLines) { char *tmp = NULL; int len = 0; - if (lines) { - tmp = lines[i]; - len = strlen(tmp); - } else if (rawLine) { - tmp = rawLine; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { - break; - } - len++; - } - if (info->protocol == TSDB_SML_LINE_PROTOCOL && tmp[0] == '#') { // this line is comment - continue; - } + if(!getLine(info, lines, rawLine, rawLineEnd, numLines, i, &tmp, &len)){ + continue; } - - uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, - info->isRawLine, numLines, info->protocol, len, info->isRawLine ? "rawdata" : tmp); - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->dataFormat) { SSmlLineInfo element = {0}; @@ -1699,7 +1714,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, info->isRawLine ? "rawdata" : tmp); + if(rawLine != NULL){ + char* print = taosMemoryCalloc(len + 1, 1); + memcpy(print, tmp, len); + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); + taosMemoryFree(print); + }else{ + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); + } return code; } if (info->reRun) { @@ -1828,7 +1850,6 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, return (TAOS_RES *)request; } info->pRequest = request; - info->isRawLine = rawLine != NULL; info->ttl = ttl; info->precision = precision; info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; @@ -1934,8 +1955,7 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi reqid); } -TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision, int32_t ttl, int64_t reqid) { +static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol){ int numLines = 0; *totalRows = 0; char *tmp = lines; @@ -1948,6 +1968,11 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int tmp = lines + i + 1; } } +} + +TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int32_t ttl, int64_t reqid) { + getRawLineLen(lines, len, totalRows, protocol); return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); } diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 2dbba38212..13c4431b62 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -210,7 +210,15 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 SSmlKv* kv = (SSmlKv*)data; if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ ret = TSDB_CODE_SML_INVALID_DATA; - uInfo("SML smlBuildCol error col not same %s", pColSchema->name); + char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); + if(tmp){ + memcpy(tmp, kv->key, kv->keyLen); + uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", + tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name); + taosMemoryFree(tmp); + }else{ + uError("SML smlBuildCol out of memory"); + } goto end; } if (kv->type == TSDB_DATA_TYPE_NCHAR) { From 19637fd148d064b87f3eb1f5644300fd4129d93e Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 18 Dec 2023 09:32:31 +0000 Subject: [PATCH 05/27] release mem check --- source/dnode/mnode/impl/src/mndCompact.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 9b35343ed2..6471fa817d 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -288,7 +288,7 @@ int32_t mndRetrieveCompact(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, tNameFromString(&name, pCompact->dbname, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&name, varDataVal(tmpBuf)); } else { - strncpy(varDataVal(tmpBuf), pCompact->dbname, strlen(pCompact->dbname) + 1); + strncpy(varDataVal(tmpBuf), pCompact->dbname, TSDB_SHOW_SQL_LEN); } varDataSetLen(tmpBuf, strlen(varDataVal(tmpBuf))); colDataSetVal(pColInfo, numOfRows, (const char *)tmpBuf, false); From ef621a06a71d55d36c7050f51247471d7cea1f40 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 18 Dec 2023 19:21:02 +0800 Subject: [PATCH 06/27] opti:logic in parse raw line & make log more specific --- source/client/src/clientSml.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c392308195..f973d165bb 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1640,25 +1640,25 @@ int32_t smlClearForRerun(SSmlHandle *info) { return TSDB_CODE_SUCCESS; } -static bool getLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, +static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLineEnd, int numLines, int i, char** tmp, int *len){ if (lines) { *tmp = lines[i]; *len = strlen(*tmp); - } else if (rawLine) { - *tmp = rawLine; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { + } else if (*rawLine) { + *tmp = *rawLine; + while (*rawLine < rawLineEnd) { + if (*((*rawLine)++) == '\n') { break; } (*len)++; } if (info->protocol == TSDB_SML_LINE_PROTOCOL && (*tmp)[0] == '#') { // this line is comment - false; + return false; } } - if(rawLine != NULL){ + if(*rawLine != NULL){ char* print = taosMemoryCalloc(*len + 1, 1); memcpy(print, *tmp, *len); uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, @@ -1692,7 +1692,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char while (i < numLines) { char *tmp = NULL; int len = 0; - if(!getLine(info, lines, rawLine, rawLineEnd, numLines, i, &tmp, &len)){ + if(!getLine(info, lines, &rawLine, rawLineEnd, numLines, i, &tmp, &len)){ continue; } if (info->protocol == TSDB_SML_LINE_PROTOCOL) { From 7e541a5f3ffff8b109e00d838f1138b1214d4b7c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 19 Dec 2023 08:49:56 +0800 Subject: [PATCH 07/27] Revert "fix: remove tags keyword from document" This reverts commit 83e2e0dffd06af1c46b70e47963a8fdb34e36e27. --- docs/en/12-taos-sql/06-select.md | 10 +++++++++- docs/zh/12-taos-sql/06-select.md | 12 +++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index f1e19a5449..f229f0d77b 100755 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -9,7 +9,7 @@ description: This document describes how to query data in TDengine. ```sql SELECT {DATABASE() | CLIENT_VERSION() | SERVER_VERSION() | SERVER_STATUS() | NOW() | TODAY() | TIMEZONE() | CURRENT_USER() | USER() } -SELECT [hints] [DISTINCT] select_list +SELECT [hints] [DISTINCT] [TAGS] select_list from_clause [WHERE condition] [partition_by_clause] @@ -227,6 +227,14 @@ The \_IROWTS pseudocolumn can only be used with INTERP function. This pseudocolu select _irowts, interp(current) from meters range('2020-01-01 10:00:00', '2020-01-01 10:30:00') every(1s) fill(linear); ``` +### TAGS Query + +The TAGS keyword returns only tag columns from all child tables when only tag columns are specified. One row containing tag columns is returned for each child table. + +```sql +SELECT TAGS tag_name [, tag_name ...] FROM stb_name +``` + ## Query Objects `FROM` can be followed by a number of tables or super tables, or can be followed by a sub-query. diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 04508ceede..1e11019ab9 100755 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -9,7 +9,7 @@ description: 查询数据的详细语法 ```sql SELECT {DATABASE() | CLIENT_VERSION() | SERVER_VERSION() | SERVER_STATUS() | NOW() | TODAY() | TIMEZONE() | CURRENT_USER() | USER() } -SELECT [hints] [DISTINCT] select_list +SELECT [hints] [DISTINCT] [TAGS] select_list from_clause [WHERE condition] [partition_by_clause] @@ -162,6 +162,16 @@ SELECT DISTINCT col_name [, col_name ...] FROM tb_name; ::: +### 标签查询 + +当查询的列只有标签列时，`TAGS` 关键字可以指定返回所有子表的标签列。每个子表只返回一行标签列。 + +返回所有子表的标签列： + +```sql +SELECT TAGS tag_name [, tag_name ...] FROM stb_name +``` + ### 结果集列名 `SELECT`子句中，如果不指定返回结果集合的列名，结果集列名称默认使用`SELECT`子句中的表达式名称作为列名称。此外，用户可使用`AS`来重命名返回结果集合中列的名称。例如： From 713d29d92aa5cb03e207d5e704bd5ac23fd22fd0 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 19 Dec 2023 08:53:45 +0800 Subject: [PATCH 08/27] doc: select tags to retrieve tags --- docs/en/12-taos-sql/06-select.md | 2 +- docs/zh/12-taos-sql/06-select.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index f229f0d77b..82acc97491 100755 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -182,7 +182,7 @@ The TBNAME pseudocolumn in a supertable contains the names of subtables within t The following SQL statement returns all unique subtable names and locations within the meters supertable: ```mysql -SELECT DISTINCT TBNAME, location FROM meters; +SELECT TAGS TBNAME, location FROM meters; ``` Use the `INS_TAGS` system table in `INFORMATION_SCHEMA` to query the information for subtables in a supertable. For example, the following statement returns the name and tag values for each subtable in the `meters` supertable. diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index 1e11019ab9..7f346bd36e 100755 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -192,7 +192,7 @@ taos> SELECT ts, ts AS primary_key_ts FROM d1001; 获取一个超级表所有的子表名及相关的标签信息： ```mysql -SELECT DISTINCT TBNAME, location FROM meters; +SELECT TAGS TBNAME, location FROM meters; ``` 建议用户使用 INFORMATION_SCHEMA 下的 INS_TAGS 系统表来查询超级表的子表标签信息，例如获取超级表 meters 所有的子表名和标签值： From 84d18005f20554ee243e6c287190ffc03595bcae Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 19 Dec 2023 11:34:15 +0800 Subject: [PATCH 09/27] fix:add control for raw data in schemaless --- source/client/src/clientSml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index f973d165bb..6bcdb4e973 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1658,7 +1658,7 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi } } - if(*rawLine != NULL){ + if(*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)){ char* print = taosMemoryCalloc(*len + 1, 1); memcpy(print, *tmp, *len); uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, From ad24c035381ee564b94b1de6cbfcf6b643ae10ef Mon Sep 17 00:00:00 2001 From: wangjiaming0909 <604227650@qq.com> Date: Tue, 19 Dec 2023 14:31:13 +0800 Subject: [PATCH 10/27] fix: session window block row index overflow --- source/libs/executor/src/timewindowoperator.c | 4 +--- tests/system-test/2-query/state_window.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0ca91f74ad..afe1921d30 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1328,6 +1328,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator SWindowRowsSup* pRowSup = &pInfo->winSup; pRowSup->numOfRows = 0; + pRowSup->startRowIndex = 0; // In case of ascending or descending order scan data, only one time window needs to be kepted for each table. TSKEY* tsList = (TSKEY*)pColInfoData->pData; @@ -1339,9 +1340,6 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator ((pRowSup->prevTs - tsList[j] >= 0) && (pRowSup->prevTs - tsList[j] <= gap))) { // The gap is less than the threshold, so it belongs to current session window that has been opened already. doKeepTuple(pRowSup, tsList[j], gid); - if (j == 0 && pRowSup->startRowIndex != 0) { - pRowSup->startRowIndex = 0; - } } else { // start a new session window SResultRow* pResult = NULL; diff --git a/tests/system-test/2-query/state_window.py b/tests/system-test/2-query/state_window.py index a211cd9dbe..2bf363f0dc 100644 --- a/tests/system-test/2-query/state_window.py +++ b/tests/system-test/2-query/state_window.py @@ -186,7 +186,25 @@ class TDTestCase: tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) tdSql.execute("select bottom(c1, 1), c2 from t0 state_window(c2) order by ts", queryTimes=1) + def test_crash_for_session_window(self): + tdSql.execute("drop database if exists test") + self.prepareTestEnv() + tdSql.execute("alter local 'queryPolicy' '3'") + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 2,2,2,2,2,2,2,2,2)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,3,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.execute("flush database test", queryTimes=1) + time.sleep(2) + tdSql.execute("insert into t0 values(now, 3,NULL,3,3,3,3,3,3,3)", queryTimes=1) + tdSql.query("select first(c2) from t0 session(ts, 1s) order by ts", queryTimes=1) + def run(self): + self.test_crash_for_session_window() self.test_crash_for_state_window1() self.test_crash_for_state_window2() self.test_crash_for_state_window3() From cfa1faf79ee21c3db436864395ce91b9bc274406 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 19 Dec 2023 15:51:27 +0800 Subject: [PATCH 11/27] test: support for windows using 'taos -s' --- tests/system-test/2-query/ts-4233.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/system-test/2-query/ts-4233.py b/tests/system-test/2-query/ts-4233.py index 9b0a2f175c..a0feb1bf48 100644 --- a/tests/system-test/2-query/ts-4233.py +++ b/tests/system-test/2-query/ts-4233.py @@ -14,7 +14,7 @@ class TDTestCase: tdSql.init(conn.cursor(), True) def checksql(self, sql): - result = os.popen("taos -s '%s'" %sql) + result = os.popen(f"taos -s \"{sql}\" ") res = result.read() print(res) if ("Query OK" in res): @@ -23,14 +23,12 @@ class TDTestCase: tdLog.exit(f"checkEqual error") def check(self): - conn = taos.connect() sql = "select 'a;b' as x" tdSql.query(f"%s" %sql) tdSql.checkRows(1) - self.checksql('select "a;b" as x\G') - self.checksql('select "a;b" as x >> /tmp/res.txt') - return + self.checksql('select \\\"a;b\\\" as x\G') + self.checksql('select \\\"a;b\\\" as x >> temp.txt') def run(self): tdSql.prepare() From d819bb4bc1b568003e1b4e328f8b5da7894e9c39 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Tue, 19 Dec 2023 17:33:50 +0800 Subject: [PATCH 12/27] test: support for windows --- tests/system-test/7-tmq/tmq_offset.py | 47 ++++++++++++++++++--------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/tests/system-test/7-tmq/tmq_offset.py b/tests/system-test/7-tmq/tmq_offset.py index 6453f452c6..8272449ccb 100644 --- a/tests/system-test/7-tmq/tmq_offset.py +++ b/tests/system-test/7-tmq/tmq_offset.py @@ -5,7 +5,7 @@ import time import socket import os import threading - +import platform from util.log import * from util.sql import * from util.cases import * @@ -21,23 +21,40 @@ class TDTestCase: tdSql.init(conn.cursor()) def run(self): - tdSql.prepare() - buildPath = tdCom.getBuildPath() - cmdStr1 = '%s/build/bin/taosBenchmark -i 50 -B 1 -t 1000 -n 100000 -y &'%(buildPath) - tdLog.info(cmdStr1) - os.system(cmdStr1) - time.sleep(15) - cmdStr2 = '%s/build/bin/tmq_offset_test &'%(buildPath) - tdLog.info(cmdStr2) - os.system(cmdStr2) + if platform.system().lower() == 'windows': + buildPath = tdCom.getBuildPath() + cmdStr1 = ' mintty -h never %s/build/bin/taosBenchmark -i 50 -B 1 -t 1000 -n 100000 -y '%(buildPath) + tdLog.info(cmdStr1) + os.system(cmdStr1) + time.sleep(15) - time.sleep(20) + cmdStr2 = ' mintty -h never %s/build/bin/tmq_offset_test '%(buildPath) + tdLog.info(cmdStr2) + os.system(cmdStr2) + time.sleep(20) - os.system("kill -9 `pgrep taosBenchmark`") - result = os.system("kill -9 `pgrep tmq_offset_test`") - if result != 0: - tdLog.exit("tmq_offset_test error!") + # tdLog.info("ps -a | grep taos | awk \'{print $2}\' | xargs kill -9") + os.system('ps -a | grep taosBenchmark | awk \'{print $2}\' | xargs kill -9') + result = os.system('ps -a | grep tmq_offset_test | awk \'{print $2}\' | xargs kill -9') + if result != 0: + tdLog.exit("tmq_offset_test error!") + else: + buildPath = tdCom.getBuildPath() + cmdStr1 = '%s/build/bin/taosBenchmark -i 50 -B 1 -t 1000 -n 100000 -y &'%(buildPath) + tdLog.info(cmdStr1) + os.system(cmdStr1) + time.sleep(15) + + cmdStr2 = '%s/build/bin/tmq_offset_test &'%(buildPath) + tdLog.info(cmdStr2) + os.system(cmdStr2) + time.sleep(20) + + os.system("kill -9 `pgrep taosBenchmark`") + result = os.system("kill -9 `pgrep tmq_offset_test`") + if result != 0: + tdLog.exit("tmq_offset_test error!") def stop(self): tdSql.close() From 455e70f4ac30307a4623281b0d6741667a1a805e Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 20 Dec 2023 11:16:33 +0800 Subject: [PATCH 13/27] docs: release 3.2.2.0 --- cmake/cmake.version | 2 +- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index e9cf31e75b..0e35fa316f 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.2.2.0.alpha") + SET(TD_VER_NUMBER "3.2.3.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index f846ede52f..f5a4789976 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.2.2.0 + + + ## 3.2.1.0 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index cfc4b92eee..b0a81e01a1 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.2.2.0 + + + ## 3.2.1.0 From 1f70f51ac8e2897abdfc4942cf5765f5cf6e753f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 20 Dec 2023 11:20:41 +0800 Subject: [PATCH 14/27] fix:sdb maxId should use int64* instead of int32* --- source/dnode/mnode/sdb/src/sdbHash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 109a3ca211..1d2e2de17d 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -184,7 +184,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); } if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT64) { - pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); + pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int64_t *)pRow->pObj)); } pSdb->tableVer[pRow->type]++; From 8160eaec90de8402c32cfcd863fa18890ff180b8 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Wed, 20 Dec 2023 16:19:21 +0800 Subject: [PATCH 15/27] test: support for windows in ci cases --- tests/pytest/client/twoClients.py | 2 +- tests/pytest/util/cluster.py | 3 +- tests/pytest/util/dnodes.py | 1 + tests/system-test/2-query/diff.py | 12 +- .../2-query/nestedQueryInterval.py | 250 ++++++------------ 5 files changed, 95 insertions(+), 173 deletions(-) diff --git a/tests/pytest/client/twoClients.py b/tests/pytest/client/twoClients.py index 1a1b36c554..cbafeccf74 100644 --- a/tests/pytest/client/twoClients.py +++ b/tests/pytest/client/twoClients.py @@ -64,7 +64,7 @@ class TwoClients: cursor2.execute("drop table t0") cursor2.execute("create table t0 using tb tags('beijing')") - tdSql.execute("insert into t0 values(now, 2, 'test')") + tdSql.execute("insert into t0 values(now+1s, 2, 'test')") tdSql.query("select * from tb") tdSql.checkRows(1) diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py index 30b70b01fc..2bf2c99604 100644 --- a/tests/pytest/util/cluster.py +++ b/tests/pytest/util/cluster.py @@ -52,10 +52,9 @@ class ConfigureyCluster: dnode.addExtraCfg("secondEp", f"{hostname}:{startPort_sec}") # configure dnoe of independent mnodes - if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == True : + if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == "True" : tdLog.info(f"set mnode:{num} supportVnodes 0") dnode.addExtraCfg("supportVnodes", 0) - # print(dnode) self.dnodes.append(dnode) return self.dnodes diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 67c3d37960..789e5866ee 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -130,6 +130,7 @@ class TDDnode: "locale": "en_US.UTF-8", "charset": "UTF-8", "asyncLog": "0", + "DebugFlag": "131", "mDebugFlag": "143", "dDebugFlag": "143", "vDebugFlag": "143", diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 10e16a690f..15f73344d3 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -88,12 +88,12 @@ class TDTestCase: tdSql.execute( f"create table {dbname}.ntb_null(ts timestamp,c1 int,c2 double,c3 float,c4 bool)") tdSql.execute(f"insert into {dbname}.ntb_null values(now, 1, 1.0, NULL, NULL)") - tdSql.execute(f"insert into {dbname}.ntb_null values(now, NULL, 2.0, 2.0, NULL)") - tdSql.execute(f"insert into {dbname}.ntb_null values(now, 2, NULL, NULL, false)") - tdSql.execute(f"insert into {dbname}.ntb_null values(now, NULL, 1.0, 1.0, NULL)") - tdSql.execute(f"insert into {dbname}.ntb_null values(now, NULL, 3.0, NULL, true)") - tdSql.execute(f"insert into {dbname}.ntb_null values(now, 3, NULL, 3.0, NULL)") - tdSql.execute(f"insert into {dbname}.ntb_null values(now, 1, NULL, NULL, true)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now+1s, NULL, 2.0, 2.0, NULL)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now+2s, 2, NULL, NULL, false)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now+3s, NULL, 1.0, 1.0, NULL)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now+4s, NULL, 3.0, NULL, true)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now+5s, 3, NULL, 3.0, NULL)") + tdSql.execute(f"insert into {dbname}.ntb_null values(now+6s, 1, NULL, NULL, true)") tdSql.query(f"select diff(c1) from {dbname}.ntb_null") tdSql.checkRows(6) diff --git a/tests/system-test/2-query/nestedQueryInterval.py b/tests/system-test/2-query/nestedQueryInterval.py index e937774c4c..c16fc03c27 100644 --- a/tests/system-test/2-query/nestedQueryInterval.py +++ b/tests/system-test/2-query/nestedQueryInterval.py @@ -1113,157 +1113,106 @@ class TDTestCase: tdLog.debug("test insert data into stable") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(2) - tdSql.checkData(0, 1, 100); - tdSql.checkData(1, 1, 200); + tdSql.checkData(0, 1, 100) + tdSql.checkData(1, 1, 200) tdSql.query(f"insert into nested.stable_1 (ts,tbname) values(now,'stable_1_1');") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(2) - tdSql.checkData(0, 1, 101); - tdSql.checkData(1, 1, 200); + tdSql.checkData(0, 1, 101) + tdSql.checkData(1, 1, 200) - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_bigint) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_smallint) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_tinyint) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_float) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_double) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_bool) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_binary) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_nchar) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_ts) values(now,'stable_1_1',1);") - tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") + qlist = ['q_int', 'q_bigint', 'q_smallint', 'q_tinyint', 'q_float', 'q_double', 'q_bool', 'q_binary', 'q_nchar', 'q_ts'] + for i in range(10): + coulmn_name = qlist[i] + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") + tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;",queryTimes=5) tdSql.checkRows(2) - tdSql.checkData(0, 1, 111); - tdSql.checkData(1, 1, 200); - - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_bigint_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_smallint_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_tinyint_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_float_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_double_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_bool_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_binary_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_nchar_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_ts_null) values(now,'stable_1_1',1);") - tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") + tdSql.checkData(0, 1, 111) + tdSql.checkData(1, 1, 200) + + q_null_list = ['q_int_null', 'q_bigint_null', 'q_smallint_null', 'q_tinyint_null', 'q_float_null', 'q_double_null', 'q_bool_null', 'q_binary_null', 'q_nchar_null', 'q_ts_null'] + for i in range(10): + coulmn_name = q_null_list[i] + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") + tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;",queryTimes=5) tdSql.checkRows(2) - tdSql.checkData(0, 1, 121); - tdSql.checkData(1, 1, 200); + tdSql.checkData(0, 1, 121) + tdSql.checkData(1, 1, 200) tdSql.query(f"insert into nested.stable_null_data (ts,tbname) values(now,'stable_null_data_1');") tdSql.query(f"select tbname,count(*) from nested.stable_null_data_1 group by tbname order by tbname;") tdSql.checkRows(1) - tdSql.checkData(0, 1, 1); - - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_bigint) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_smallint) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_tinyint) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_float) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_double) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_bool) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_binary) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_nchar) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_ts) values(now,'stable_null_data_1',1);") + tdSql.checkData(0, 1, 1) + + for i in range(10): + coulmn_name = qlist[i] + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") + + tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;",queryTimes=5) + tdSql.checkRows(1) + tdSql.checkData(0, 1, 11) + + for i in range(10): + coulmn_name = q_null_list[i] + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") + tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) - tdSql.checkData(0, 1, 11); - - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_bigint_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_smallint_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_tinyint_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_float_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_double_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_bool_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_binary_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_nchar_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_ts_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") - tdSql.checkRows(1) - tdSql.checkData(0, 1, 21); - + tdSql.checkData(0, 1, 21) tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname) values(now,'stable_null_childtable_1');") tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) - tdSql.checkData(0, 1, 1); - - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_bigint) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_smallint) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_tinyint) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_float) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_double) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_bool) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_binary) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_nchar) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_ts) values(now,'stable_null_childtable_1',1);") + tdSql.checkData(0, 1, 1) + + for i in range(10): + coulmn_name = qlist[i] + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) - tdSql.checkData(0, 1, 11); - - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_bigint_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_smallint_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_tinyint_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_float_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_double_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_bool_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_binary_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_nchar_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_ts_null) values(now,'stable_null_childtable_1',1);") + tdSql.checkData(0, 1, 11) + + for i in range(10): + coulmn_name = q_null_list[i] + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) - tdSql.checkData(0, 1, 21); + tdSql.checkData(0, 1, 21) def TS_3932_flushdb(self): tdLog.debug("test flush db and insert data into stable") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(2) - tdSql.checkData(0, 1, 121); - tdSql.checkData(1, 1, 200); - + tdSql.checkData(0, 1, 121) + tdSql.checkData(1, 1, 200) + + qlist = ['q_int', 'q_bigint', 'q_smallint', 'q_tinyint', 'q_float', 'q_double', 'q_bool', 'q_binary', 'q_nchar', 'q_ts'] + q_null_list = ['q_int_null', 'q_bigint_null', 'q_smallint_null', 'q_tinyint_null', 'q_float_null', 'q_double_null', 'q_bool_null', 'q_binary_null', 'q_nchar_null', 'q_ts_null'] tdSql.query(f"insert into nested.stable_1 (ts,tbname) values(now,'stable_1_1');") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(2) - tdSql.checkData(0, 1, 122); - tdSql.checkData(1, 1, 200); - - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_bigint) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_smallint) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_tinyint) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_float) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_double) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_bool) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_binary) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_nchar) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_ts) values(now,'stable_1_1',1);") + tdSql.checkData(0, 1, 122) + tdSql.checkData(1, 1, 200) + + for i in range(10): + coulmn_name = qlist[i] + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(2) - tdSql.checkData(0, 1, 132); - tdSql.checkData(1, 1, 200); - - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_bigint_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_smallint_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_tinyint_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_float_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_double_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_bool_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_binary_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_nchar_null) values(now,'stable_1_1',1);") - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_ts_null) values(now,'stable_1_1',1);") + tdSql.checkData(0, 1, 132) + tdSql.checkData(1, 1, 200) + + for i in range(10): + coulmn_name = q_null_list[i] + tdSql.execute(f"insert into nested.stable_1 (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_1_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_1 group by tbname order by tbname;") tdSql.checkRows(2) - tdSql.checkData(0, 1, 142); - tdSql.checkData(1, 1, 200); + tdSql.checkData(0, 1, 142) + tdSql.checkData(1, 1, 200) - tdSql.query(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'stable_1_1',1) \ + tdSql.execute(f"insert into nested.stable_1 (ts,tbname,q_int) values(now,'stable_1_1',1) \ nested.stable_1 (ts,tbname,q_bigint) values(now+1a,'stable_1_1',1)\ nested.stable_1 (ts,tbname,q_smallint) values(now+2a,'stable_1_1',1)\ nested.stable_1 (ts,tbname,q_tinyint) values(now+3a,'stable_1_1',1)\ @@ -1282,34 +1231,21 @@ class TDTestCase: tdSql.query(f"select tbname,count(*) from nested.stable_null_data_1 group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 22); - - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_bigint) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_smallint) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_tinyint) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_float) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_double) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_bool) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_binary) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_nchar) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_ts) values(now,'stable_null_data_1',1);") + + for i in range(10): + coulmn_name = qlist[i] + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) - tdSql.checkData(0, 1, 32); - - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_bigint_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_smallint_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_tinyint_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_float_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_double_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_bool_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_binary_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_nchar_null) values(now,'stable_null_data_1',1);") - tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_ts_null) values(now,'stable_null_data_1',1);") + tdSql.checkData(0, 1, 32) + + for i in range(10): + coulmn_name = q_null_list[i] + tdSql.execute(f"insert into nested.stable_null_data (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_data_1',1);") + tdSql.query(f"select tbname,count(*) from nested.stable_null_data group by tbname order by tbname;") tdSql.checkRows(1) - tdSql.checkData(0, 1, 42); + tdSql.checkData(0, 1, 42) tdSql.query(f"insert into nested.stable_null_data (ts,tbname,q_int) values(now,'stable_null_data_1',1) \ nested.stable_null_data (ts,tbname,q_bigint) values(now+1a,'stable_null_data_1',1)\ @@ -1330,32 +1266,18 @@ class TDTestCase: tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname) values(now,'stable_null_childtable_1');") tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) - tdSql.checkData(0, 1, 22); - - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_bigint) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_smallint) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_tinyint) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_float) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_double) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_bool) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_binary) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_nchar) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_ts) values(now,'stable_null_childtable_1',1);") + tdSql.checkData(0, 1, 22) + + for i in range(10): + coulmn_name = qlist[i] + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) - tdSql.checkData(0, 1, 32); - - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_int_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_bigint_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_smallint_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_tinyint_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_float_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_double_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_bool_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_binary_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_nchar_null) values(now,'stable_null_childtable_1',1);") - tdSql.query(f"insert into nested.stable_null_childtable (ts,tbname,q_ts_null) values(now,'stable_null_childtable_1',1);") + tdSql.checkData(0, 1, 32) + + for i in range(10): + coulmn_name = q_null_list[i] + tdSql.execute(f"insert into nested.stable_null_childtable (ts, tbname, {coulmn_name}) values(now+{i}s,'stable_null_childtable_1',1);") tdSql.query(f"select tbname,count(*) from nested.stable_null_childtable group by tbname order by tbname;") tdSql.checkRows(1) tdSql.checkData(0, 1, 42); From d6297e26ce089ce4ff2d97db792719b524af0c6d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 20 Dec 2023 16:56:12 +0800 Subject: [PATCH 16/27] init backend mutex --- source/libs/stream/src/streamMeta.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index ce391eeadb..6c5b1ef994 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -394,6 +394,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); pMeta->bkdChkptMgt = bkdMgtCreate(tpath); + taosThreadMutexInit(&pMeta->backendMutex, NULL); return pMeta; From 3c7170a9746d8189787a0567515de36a3f3f9d5d Mon Sep 17 00:00:00 2001 From: haoranchen Date: Wed, 20 Dec 2023 18:45:33 +0800 Subject: [PATCH 17/27] test:modify debugflag 131 in db.py and default dnode --- tests/system-test/2-query/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index 0246626e40..a48b4cf3bb 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -52,7 +52,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, i + 1) tdSql.checkData(0, 1, 'debugFlag') - tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 2, 131) tdSql.query("show dnode 1 variables like '%debugFlag'") tdSql.checkRows(23) From b0247f649b415504677a6a5171e50820aa7666ab Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Dec 2023 13:41:49 +0800 Subject: [PATCH 18/27] coverage : comment trow.c --- source/common/src/trow.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index e2da4d166e..530de2a4bc 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -17,6 +17,7 @@ #include "trow.h" #include "tlog.h" +#ifdef BUILD_NO_CALL const uint8_t tdVTypeByte[2][3] = {{ // 2 bits TD_VTYPE_NORM_BYTE_II, @@ -1079,3 +1080,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV } } } +#endif \ No newline at end of file From 9d4a75b0beaddb5744f34bae3c4a685d2e425f27 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Dec 2023 14:22:22 +0800 Subject: [PATCH 19/27] coverage: vnodeSrv.c tmsg.c trow.c --- source/common/src/tmsg.c | 2 + source/common/src/trow.c | 423 +++++++++++++------------- source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +- 3 files changed, 218 insertions(+), 210 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8cf48d41dc..3ff814dda9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -143,6 +143,7 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { } } +#ifdef BUILD_NO_CALL int32_t tPrintFixedSchemaSubmitReq(SSubmitReq *pReq, STSchema *pTschema) { SSubmitMsgIter msgIter = {0}; if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1; @@ -161,6 +162,7 @@ int32_t tPrintFixedSchemaSubmitReq(SSubmitReq *pReq, STSchema *pTschema) { } return 0; } +#endif int32_t tEncodeSEpSet(SEncoder *pEncoder, const SEpSet *pEp) { if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 530de2a4bc..b91562be7a 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -17,6 +17,220 @@ #include "trow.h" #include "tlog.h" +static bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); +static bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); + +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) { + pIter->pSchema = pSchema; + pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; +} + +void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) { +#ifdef TD_SUPPORT_BITMAP + switch (rowType) { + case TD_ROW_TP: + return tdGetBitmapAddrTp(pRow, flen); + case TD_ROW_KV: + return tdGetBitmapAddrKv(pRow, nKvCols); + default: + break; + } +#endif + return NULL; +} + +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { + pIter->pRow = pRow; + pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); + pIter->offset = 0; + pIter->colIdx = 0; // PRIMARYKEY_TIMESTAMP_COL_ID; + pIter->kvIdx = 0; +} + +bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + if (TD_IS_TP_ROW(pIter->pRow)) { + STColumn *pCol = NULL; + STSchema *pSchema = pIter->pSchema; + while (pIter->colIdx < pSchema->numOfCols) { + pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key + if (colId == pCol->colId) { + break; + } else if (pCol->colId < colId) { + ++pIter->colIdx; + continue; + } else { + return false; + } + } + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); + ++pIter->colIdx; + } else if (TD_IS_KV_ROW(pIter->pRow)) { + return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); + } else { + pVal->valType = TD_VTYPE_NONE; + terrno = TSDB_CODE_INVALID_PARA; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + } + return true; +} + +bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + if (pRow->statis == 0) { + pVal->valType = TD_VTYPE_NORM; + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + return true; + } + + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + return true; + } + + if (pVal->valType == TD_VTYPE_NORM) { + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + } + + return true; +} + +int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { + if (!pBitmap || colIdx < 0) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS; + int16_t nOffset = colIdx & TD_VTYPE_OPTR; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pValType = (((*pDestByte) & 0xC0) >> 6); + break; + case 1: + *pValType = (((*pDestByte) & 0x30) >> 4); + break; + case 2: + *pValType = (((*pDestByte) & 0x0C) >> 2); + break; + case 3: + *pValType = ((*pDestByte) & 0x03); + break; + default: + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { + if (!pBitmap || colIdx < 0) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; + int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pValType = (((*pDestByte) & 0x80) >> 7); + break; + case 1: + *pValType = (((*pDestByte) & 0x40) >> 6); + break; + case 2: + *pValType = (((*pDestByte) & 0x20) >> 5); + break; + case 3: + *pValType = (((*pDestByte) & 0x10) >> 4); + break; + case 4: + *pValType = (((*pDestByte) & 0x08) >> 3); + break; + case 5: + *pValType = (((*pDestByte) & 0x04) >> 2); + break; + case 6: + *pValType = (((*pDestByte) & 0x02) >> 1); + break; + case 7: + *pValType = ((*pDestByte) & 0x01); + break; + default: + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { + switch (bitmapMode) { + case 0: + tdGetBitmapValTypeII(pBitmap, colIdx, pValType); + break; + case -1: + case 1: + tdGetBitmapValTypeI(pBitmap, colIdx, pValType); + break; + default: + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + SKvRowIdx *pKvIdx = NULL; + bool colFound = false; + col_id_t kvNCols = tdRowGetNCols(pRow) - 1; + void *pColIdx = TD_ROW_COL_IDX(pRow); + while (*nIdx < kvNCols) { + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, *nIdx * sizeof(SKvRowIdx)); + if (pKvIdx->colId == colId) { + ++(*nIdx); + pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); + colFound = true; + break; + } else if (pKvIdx->colId > colId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + ++(*nIdx); + } + } + + if (!colFound) { + if (colId <= pIter->maxColId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + return false; + } + } + + if (tdGetBitmapValType(pIter->pBitmap, pIter->kvIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + } + + return true; +} + #ifdef BUILD_NO_CALL const uint8_t tdVTypeByte[2][3] = {{ // 2 bits @@ -35,8 +249,6 @@ const uint8_t tdVTypeByte[2][3] = {{ // declaration static uint8_t tdGetBitmapByte(uint8_t byte); -static bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); -static bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); static bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, SCellVal *pVal); static bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal); @@ -200,38 +412,6 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } -bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pVal->val = &pIter->pRow->ts; - pVal->valType = TD_VTYPE_NORM; - return true; - } - - if (TD_IS_TP_ROW(pIter->pRow)) { - STColumn *pCol = NULL; - STSchema *pSchema = pIter->pSchema; - while (pIter->colIdx < pSchema->numOfCols) { - pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key - if (colId == pCol->colId) { - break; - } else if (pCol->colId < colId) { - ++pIter->colIdx; - continue; - } else { - return false; - } - } - tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset, pVal); - ++pIter->colIdx; - } else if (TD_IS_KV_ROW(pIter->pRow)) { - return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); - } else { - pVal->valType = TD_VTYPE_NONE; - terrno = TSDB_CODE_INVALID_PARA; - if (COL_REACH_END(colId, pIter->maxColId)) return false; - } - return true; -} bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { if (pIter->colIdx >= pIter->pSchema->numOfCols) { @@ -259,71 +439,6 @@ bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { return true; } -bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - if (pRow->statis == 0) { - pVal->valType = TD_VTYPE_NORM; - if (IS_VAR_DATA_TYPE(colType)) { - pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - return true; - } - - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - return true; - } - - if (pVal->valType == TD_VTYPE_NORM) { - if (IS_VAR_DATA_TYPE(colType)) { - pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - } - - return true; -} - -bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - SKvRowIdx *pKvIdx = NULL; - bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow) - 1; - void *pColIdx = TD_ROW_COL_IDX(pRow); - while (*nIdx < kvNCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, *nIdx * sizeof(SKvRowIdx)); - if (pKvIdx->colId == colId) { - ++(*nIdx); - pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); - colFound = true; - break; - } else if (pKvIdx->colId > colId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - ++(*nIdx); - } - } - - if (!colFound) { - if (colId <= pIter->maxColId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - return false; - } - } - - if (tdGetBitmapValType(pIter->pBitmap, pIter->kvIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } - - return true; -} - int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { STColumn *pTColumn; SColVal *pColVal; @@ -491,76 +606,6 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell return true; } -int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { - if (!pBitmap || colIdx < 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS; - int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pValType = (((*pDestByte) & 0xC0) >> 6); - break; - case 1: - *pValType = (((*pDestByte) & 0x30) >> 4); - break; - case 2: - *pValType = (((*pDestByte) & 0x0C) >> 2); - break; - case 3: - *pValType = ((*pDestByte) & 0x03); - break; - default: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { - if (!pBitmap || colIdx < 0) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; - int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pValType = (((*pDestByte) & 0x80) >> 7); - break; - case 1: - *pValType = (((*pDestByte) & 0x40) >> 6); - break; - case 2: - *pValType = (((*pDestByte) & 0x20) >> 5); - break; - case 3: - *pValType = (((*pDestByte) & 0x10) >> 4); - break; - case 4: - *pValType = (((*pDestByte) & 0x08) >> 3); - break; - case 5: - *pValType = (((*pDestByte) & 0x04) >> 2); - break; - case 6: - *pValType = (((*pDestByte) & 0x02) >> 1); - break; - case 7: - *pValType = ((*pDestByte) & 0x01); - break; - default: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { terrno = TSDB_CODE_INVALID_PARA; @@ -945,21 +990,6 @@ int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, return TSDB_CODE_SUCCESS; } -int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { - switch (bitmapMode) { - case 0: - tdGetBitmapValTypeII(pBitmap, colIdx, pValType); - break; - case -1: - case 1: - tdGetBitmapValTypeI(pBitmap, colIdx, pValType); - break; - default: - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} #if 0 bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { TDRowValT valType = 0; @@ -1021,32 +1051,7 @@ int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int return TSDB_CODE_SUCCESS; } -void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) { -#ifdef TD_SUPPORT_BITMAP - switch (rowType) { - case TD_ROW_TP: - return tdGetBitmapAddrTp(pRow, flen); - case TD_ROW_KV: - return tdGetBitmapAddrKv(pRow, nKvCols); - default: - break; - } -#endif - return NULL; -} -void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { - pIter->pRow = pRow; - pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); - pIter->offset = 0; - pIter->colIdx = 0; // PRIMARYKEY_TIMESTAMP_COL_ID; - pIter->kvIdx = 0; -} - -void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) { - pIter->pSchema = pSchema; - pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; -} void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColVal) { STColumn *pTColumn = &pTSchema->columns[iCol]; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index afb8f962b3..db807d000b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1266,6 +1266,7 @@ _exit: return 0; } +#ifdef BUILD_NO_CALL static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, const char *tags) { SSubmitBlkIter blkIter = {0}; @@ -1296,7 +1297,7 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, return TSDB_CODE_SUCCESS; } - +#endif typedef struct SSubmitReqConvertCxt { SSubmitMsgIter msgIter; SSubmitBlk *pBlock; From 1fa6cb8f69de8692f7dc8b0283c01baa9adb85a1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Dec 2023 15:10:21 +0800 Subject: [PATCH 20/27] coverage: isRowEntryCompleted had been called 5200w hits --- include/libs/function/function.h | 6 +++--- source/libs/function/src/tfunctionInt.c | 8 -------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 003e9b900a..ffaad69373 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -237,9 +237,9 @@ struct SScalarParam { int32_t numOfQualified; // number of qualified elements in the final results }; -void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell); -bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry); -bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry); +#define cleanupResultRowEntry(p) p->initialized = false +#define isRowEntryCompleted(p) (p->complete) +#define isRowEntryInitialized(p) (p->initialized) typedef struct SPoint { int64_t key; diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c index e8041d1704..3da5d63fa1 100644 --- a/source/libs/function/src/tfunctionInt.c +++ b/source/libs/function/src/tfunctionInt.c @@ -28,8 +28,6 @@ #include "ttszip.h" #include "tudf.h" -void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { pCell->initialized = false; } - int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock) { int32_t maxRows = 0; @@ -59,9 +57,3 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock pResBlock->info.rows = maxRows; return maxRows; } - -bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { - return pEntry->complete; -} - -bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { return pEntry->initialized; } From 13851a60a4ffcb9dacc4d026e627f6a494eae7c0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 21 Dec 2023 15:26:41 +0800 Subject: [PATCH 21/27] fix:[TD-27979] init mutux --- source/libs/stream/src/streamMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index ce391eeadb..dc69c392cf 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -394,7 +394,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->qHandle = taosInitScheduler(32, 1, "stream-chkp", NULL); pMeta->bkdChkptMgt = bkdMgtCreate(tpath); - + taosThreadMutexInit(&pMeta->backendMutex, NULL); return pMeta; _err: From 094c6cdfb4e4c2eb7f9f4f1b8ee767cff3f761fc Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Dec 2023 16:31:35 +0800 Subject: [PATCH 22/27] fix: the pre-allocated column has no data to copy --- source/libs/executor/src/tsort.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 1ebc7ad3c6..205cd7d3ef 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -475,6 +475,7 @@ static void appendOneRowToDataBlock(SSDataBlock* pBlock, const SSDataBlock* pSou if (isNull) { colDataSetVal(pColInfo, pBlock->info.rows, NULL, true); } else { + if (!pSrcColInfo->pData) continue; char* pData = colDataGetData(pSrcColInfo, *rowIndex); colDataSetVal(pColInfo, pBlock->info.rows, pData, false); } @@ -900,7 +901,7 @@ static int32_t getPageBufIncForRow(SSDataBlock* blk, int32_t row, int32_t rowIdx for (int32_t i = 0; i < numCols; ++i) { SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(blk->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - if (pColInfoData->varmeta.offset[row] != -1) { + if ((pColInfoData->varmeta.offset[row] != -1) && (pColInfoData->pData)) { char* p = colDataGetData(pColInfoData, row); sz += varDataTLen(p); } @@ -970,7 +971,6 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1]; appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); nMergedRows += pHandle->pDataBlock->info.rows; - blockDataCleanup(pHandle->pDataBlock); blkPgSz = pgHeaderSz; bufInc = getPageBufIncForRow(minBlk, minRow, 0); From c5e39b8a81d7fd1db1fd3e6dac3fbe056d2d37a1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 21 Dec 2023 16:52:27 +0800 Subject: [PATCH 23/27] fix: add test case --- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/sort-pre-cols.sim | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/script/tsim/query/sort-pre-cols.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 8edfb352ab..bcdd143cfc 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1069,6 +1069,7 @@ e ,,y,script,./test.sh -f tsim/query/unionall_as_table.sim ,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim +,,y,script,./test.sh -f tsim/query/sort-pre-cols.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/query/groupby_distinct.sim ,,y,script,./test.sh -f tsim/query/event.sim diff --git a/tests/script/tsim/query/sort-pre-cols.sim b/tests/script/tsim/query/sort-pre-cols.sim new file mode 100644 index 0000000000..ef69725d87 --- /dev/null +++ b/tests/script/tsim/query/sort-pre-cols.sim @@ -0,0 +1,17 @@ + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database d +sql use d +sql create table st(ts timestamp, v int) tags(lj json) +sql insert into ct1 using st tags('{"instance":"200"}') values(now, 1)(now+1s, 2); +sql insert into ct2 using st tags('{"instance":"200"}') values(now+2s, 3)(now+3s, 4); +sql select to_char(ts, 'yyyy-mm-dd hh24:mi:ss') as time, irate(v) from st group by to_char(ts, 'yyyy-mm-dd hh24:mi:ss'), lj->'instance' order by time; +print $data01 +if $data01 != 0.000000000 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT From bcb3518622059fb915e716dcbc1227eb517193d8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 21 Dec 2023 17:46:38 +0800 Subject: [PATCH 24/27] fix: shell autotab extend to save 95 visible character --- tools/shell/inc/shellTire.h | 4 ++-- tools/shell/src/shellTire.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/shell/inc/shellTire.h b/tools/shell/inc/shellTire.h index e87c3ee4f3..472f604a2c 100644 --- a/tools/shell/inc/shellTire.h +++ b/tools/shell/inc/shellTire.h @@ -19,8 +19,8 @@ // // The prefix search tree is a efficient storage words and search words tree, it support 95 visible ascii code character // -#define FIRST_ASCII 40 // first visible char is '0' -#define LAST_ASCII 122 // last visilbe char is 'z' +#define FIRST_ASCII 32 // first visible char is '0' +#define LAST_ASCII 126 // last visilbe char is 'z' // capacity save char is 95 #define CHAR_CNT (LAST_ASCII - FIRST_ASCII + 1) diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index a8726f3126..8aa67457f2 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -95,7 +95,7 @@ bool insertToTree(STire* tire, char* word, int len) { STireNode** nodes = tire->root.d; for (int i = 0; i < len; i++) { m = word[i] - FIRST_ASCII; - if (m < 0 || m > CHAR_CNT) { + if (m < 0 || m >= CHAR_CNT) { return false; } From 9d3eb02f3c8faca8787dd2448d4474b1f659768b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 21 Dec 2023 17:53:44 +0800 Subject: [PATCH 25/27] fix(tsdb/cache): plus 1 to nCols to load stt data --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3ca2846d07..0f5511d658 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1875,7 +1875,7 @@ static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb .backward = 1, .pSttFileBlockIterArray = pr->pLDataIterArray, .pCols = aCols, - .numOfCols = nCols, + .numOfCols = nCols + 1, .loadTombFn = loadSttTomb, .pReader = pr, .idstr = pr->idstr, From 913fda4a36c29c86d03578f45ee7046ece7e2db6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 21 Dec 2023 18:52:25 +0800 Subject: [PATCH 26/27] fix snode crash --- source/dnode/snode/src/snode.c | 77 ++++++++++++++++------------------ 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 21d813c7c0..79a89cb56e 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -19,26 +19,24 @@ #include "tqCommon.h" #include "tuuid.h" -#define sndError(...) \ - do { \ - if (sndDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("SND ERROR ", DEBUG_ERROR, sndDebugFlag, __VA_ARGS__); \ - } \ - } while (0) +// clang-format off +#define sndError(...) do { if (sndDebugFlag & DEBUG_ERROR) {taosPrintLog("SND ERROR ", DEBUG_ERROR, sndDebugFlag, __VA_ARGS__);}} while (0) +#define sndInfo(...) do { if (sndDebugFlag & DEBUG_INFO) { taosPrintLog("SND INFO ", DEBUG_INFO, sndDebugFlag, __VA_ARGS__);}} while (0) +#define sndDebug(...) do { if (sndDebugFlag & DEBUG_DEBUG) { taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__);}} while (0) +// clang-format on -#define sndInfo(...) \ - do { \ - if (sndDebugFlag & DEBUG_INFO) { \ - taosPrintLog("SND INFO ", DEBUG_INFO, sndDebugFlag, __VA_ARGS__); \ - } \ - } while (0) - -#define sndDebug(...) \ - do { \ - if (sndDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__); \ - } \ - } while (0) +static STaskId replaceStreamTaskId(SStreamTask *pTask) { + ASSERT(pTask->info.fillHistory); + STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; + pTask->id.streamId = pTask->streamTaskId.streamId; + pTask->id.taskId = pTask->streamTaskId.taskId; + return id; +} +static void restoreStreamTaskId(SStreamTask *pTask, STaskId *pId) { + ASSERT(pTask->info.fillHistory); + pTask->id.taskId = pId->taskId; + pTask->id.streamId = pId->streamId; +} int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->upstreamInfo.pList) != 0); @@ -50,23 +48,22 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer streamTaskOpenAllUpstreamInput(pTask); - SStreamTask *pSateTask = pTask; - SStreamTask task = {0}; + STaskId taskId = {0}; if (pTask->info.fillHistory) { - task.id.streamId = pTask->streamTaskId.streamId; - task.id.taskId = pTask->streamTaskId.taskId; - task.pMeta = pTask->pMeta; - pSateTask = &task; + taskId = replaceStreamTaskId(pTask); } - pTask->pState = streamStateOpen(pSnode->path, pSateTask, false, -1, -1); + pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); if (pTask->pState == NULL) { sndError("s-task:%s failed to open state for task", pTask->id.idStr); return -1; } else { sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } - + + if (pTask->info.fillHistory) { + restoreStreamTaskId(pTask, &taskId); + } int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList); SReadHandle handle = { @@ -90,8 +87,8 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer // checkpoint ver is the kept version, handled data should be the next version. if (pTask->chkInfo.checkpointId != 0) { pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1; - sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, pTask->id.idStr, - pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); + sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, + pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } char *p = NULL; @@ -99,18 +96,18 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer if (pTask->info.fillHistory) { sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 - " nextProcessVer:%" PRId64 - " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms", - SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, - pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, - (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam); + " nextProcessVer:%" PRId64 + " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms", + SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, + pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, + (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam); } else { sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 - " nextProcessVer:%" PRId64 - " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms", - SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, - pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, - (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam); + " nextProcessVer:%" PRId64 + " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms", + SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, + pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, + (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam); } return 0; } @@ -149,7 +146,7 @@ FAIL: return NULL; } -int32_t sndInit(SSnode * pSnode) { +int32_t sndInit(SSnode *pSnode) { resetStreamTaskStatus(pSnode->pMeta); startStreamTasks(pSnode->pMeta); return 0; From 16ad59fb18976b468b3de6b3a1ecf93ff642926d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 22 Dec 2023 12:12:15 +0800 Subject: [PATCH 27/27] fix(tsdb/cache): prepend ts col to load stt --- source/dnode/vnode/src/tsdb/tsdbCache.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 0f5511d658..d7fea4956d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -885,9 +885,17 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr int32_t code = 0; rocksdb_writebatch_t *wb = NULL; SArray *pTmpColArray = NULL; - int num_keys = TARRAY_SIZE(remainCols); - int16_t *aCols = taosMemoryMalloc(num_keys * sizeof(int16_t)); - int16_t *slotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); + + SIdxKey *idxKey = taosArrayGet(remainCols, 0); + if (idxKey->key.cid != PRIMARYKEY_TIMESTAMP_COL_ID) { + SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = PRIMARYKEY_TIMESTAMP_COL_ID}; + + taosArrayInsert(remainCols, 0, &(SIdxKey){0, *key}); + } + + int num_keys = TARRAY_SIZE(remainCols); + int16_t *aCols = taosMemoryMalloc(num_keys * sizeof(int16_t)); + int16_t *slotIds = taosMemoryMalloc(num_keys * sizeof(int16_t)); for (int i = 0; i < num_keys; ++i) { SIdxKey *idxKey = taosArrayGet(remainCols, i); @@ -1875,7 +1883,7 @@ static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb .backward = 1, .pSttFileBlockIterArray = pr->pLDataIterArray, .pCols = aCols, - .numOfCols = nCols + 1, + .numOfCols = nCols, .loadTombFn = loadSttTomb, .pReader = pr, .idstr = pr->idstr,