From bc98324c1df74e858ce0f742bbae9e01c939283d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 3 Feb 2023 16:38:09 +0800 Subject: [PATCH 1/6] fix: correct ctg job error code --- source/libs/catalog/src/ctgAsync.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 325d6e0e46..f84f8e2917 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -906,9 +906,14 @@ int32_t ctgCallUserCb(void* param) { } void ctgUpdateJobErrCode(SCtgJob* pJob, int32_t errCode) { - if (!NEED_CLIENT_REFRESH_VG_ERROR(errCode) || errCode == TSDB_CODE_SUCCESS) return; + if (errCode == TSDB_CODE_SUCCESS) return; - atomic_store_32(&pJob->jobResCode, errCode); + if (NEED_CLIENT_HANDLE_ERROR(errCode)) { + atomic_store_32(&pJob->jobResCode, errCode); + } else if (0 != atomic_val_compare_exchange_32(&pJob->jobResCode, 0, errCode)) { + return; + } + qDebug("QID:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); return; } From 78d72a7db262dbdea6d30156bae229bd59c6c35c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 23 Feb 2023 14:21:14 +0800 Subject: [PATCH 2/6] fix: alter column length too big issue --- source/libs/parser/src/parTranslater.c | 22 ++++++++++++++++------ source/util/src/terror.c | 2 +- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0d052846f7..8b3cfd105b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5012,14 +5012,24 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); } - if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType && - pTableMeta->tableInfo.rowSize + calcTypeBytes(pStmt->dataType) - pSchema->bytes > TSDB_MAX_BYTES_PER_ROW) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW); + if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType) { + if (calcTypeBytes(pStmt->dataType) > TSDB_MAX_FIELD_LEN) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + } + + if (pTableMeta->tableInfo.rowSize + calcTypeBytes(pStmt->dataType) - pSchema->bytes > TSDB_MAX_BYTES_PER_ROW) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW); + } } - if (TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType && - tagsLen + calcTypeBytes(pStmt->dataType) - pSchema->bytes > TSDB_MAX_TAGS_LEN) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_LENGTH, TSDB_MAX_TAGS_LEN); + if (TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) { + if (calcTypeBytes(pStmt->dataType) > TSDB_MAX_FIELD_LEN) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN); + } + + if (tagsLen + calcTypeBytes(pStmt->dataType) - pSchema->bytes > TSDB_MAX_TAGS_LEN) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_LENGTH, TSDB_MAX_TAGS_LEN); + } } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c07fa88af5..b85035ffcf 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -514,7 +514,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ROW_LENGTH, "Row length exceeds TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FIRST_COLUMN, "First column must be timestamp") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN, "Invalid binary/nchar column length") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN, "Invalid binary/nchar column/tag length") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TAGS_NUM, "Invalid number of tag columns") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PERMISSION_DENIED, "Permission denied") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Invalid stream query") From 25c555a50226cdd751d2f57f541b9adbf7ab8289 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 23 Feb 2023 14:38:57 +0800 Subject: [PATCH 3/6] fix: refresh sys db vgroups after creating database --- source/client/src/clientMsgHandler.c | 16 ++++++++++++++++ tests/script/tsim/catalog/alterInCurrent.sim | 15 +++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index f414c7e92f..ed54144858 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -163,6 +163,22 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pEpSet); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); + } else { + struct SCatalog* pCatalog = NULL; + int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (TSDB_CODE_SUCCESS == code) { + STscObj* pTscObj = pRequest->pTscObj; + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + char dbFName[TSDB_DB_FNAME_LEN]; + snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_INFORMATION_SCHEMA_DB); + catalogRefreshDBVgInfo(pCatalog, &conn, dbFName); + snprintf(dbFName, sizeof(dbFName) - 1, "%d.%s", pTscObj->acctId, TSDB_PERFORMANCE_SCHEMA_DB); + catalogRefreshDBVgInfo(pCatalog, &conn, dbFName); + } } if (pRequest->body.queryFp) { diff --git a/tests/script/tsim/catalog/alterInCurrent.sim b/tests/script/tsim/catalog/alterInCurrent.sim index 3cb337bbe1..521858c368 100644 --- a/tests/script/tsim/catalog/alterInCurrent.sim +++ b/tests/script/tsim/catalog/alterInCurrent.sim @@ -67,4 +67,19 @@ sql insert into t1 values (1591060628000, 1); sql alter table st1 drop tag t2; sql create table t2 using st1 tags(2); +print ======== drop tag in super table +sql create database if not exists aaa; +sql select table_name, db_name from information_schema.ins_tables t where t.db_name like 'aaa'; +if $rows != 0 then + return -1 +endi +sql drop database if exists foo; +sql create database if not exists foo; +sql create table foo.t(ts timestamp,name varchar(20)); +sql create table foo.xt(ts timestamp,name varchar(20)); +sql select table_name, db_name from information_schema.ins_tables t where t.db_name like 'foo'; +if $rows != 2 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 8ea6b545b8037a0f9f77f51886811cebb1983c57 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 23 Feb 2023 15:04:31 +0800 Subject: [PATCH 4/6] fix: restore catalog job error --- source/libs/catalog/src/ctgAsync.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 1276f2796f..89e92b0cc8 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -906,14 +906,9 @@ int32_t ctgCallUserCb(void* param) { } void ctgUpdateJobErrCode(SCtgJob* pJob, int32_t errCode) { - if (errCode == TSDB_CODE_SUCCESS) return; + if (!NEED_CLIENT_REFRESH_VG_ERROR(errCode) || errCode == TSDB_CODE_SUCCESS) return; - if (NEED_CLIENT_HANDLE_ERROR(errCode)) { - atomic_store_32(&pJob->jobResCode, errCode); - } else if (0 != atomic_val_compare_exchange_32(&pJob->jobResCode, 0, errCode)) { - return; - } - + atomic_store_32(&pJob->jobResCode, errCode); qDebug("QID:0x%" PRIx64 " ctg job errCode updated to %s", pJob->queryId, tstrerror(errCode)); return; } From f6acf035c991d76eded48d3a0f6e7ff66c389417 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 24 Feb 2023 11:37:16 +0800 Subject: [PATCH 5/6] fix: sys db vgroup update issue --- include/libs/qcom/query.h | 8 ++- source/client/src/clientHb.c | 84 +++++++++++++++++++--------- tests/system-test/7-tmq/tmq_taosx.py | 1 + 3 files changed, 63 insertions(+), 30 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 5b640dce92..f2f7ac5699 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -26,6 +26,7 @@ extern "C" { #include "tlog.h" #include "tmsg.h" #include "tmsgcb.h" +#include "systable.h" typedef enum { JOB_TASK_STATUS_NULL = 0, @@ -284,9 +285,10 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define REQUEST_TOTAL_EXEC_TIMES 2 -#define IS_SYS_DBNAME(_dbname) \ - (((*(_dbname) == 'i') && (0 == strcmp(_dbname, TSDB_INFORMATION_SCHEMA_DB))) || \ - ((*(_dbname) == 'p') && (0 == strcmp(_dbname, TSDB_PERFORMANCE_SCHEMA_DB)))) +#define IS_INFORMATION_SCHEMA_DB(_name) ((*(_name) == 'i') && (0 == strcmp(_name, TSDB_INFORMATION_SCHEMA_DB))) +#define IS_PERFORMANCE_SCHEMA_DB(_name) ((*(_name) == 'p') && (0 == strcmp(_name, TSDB_PERFORMANCE_SCHEMA_DB))) + +#define IS_SYS_DBNAME(_dbname) (IS_INFORMATION_SCHEMA_DB(_dbname) || IS_PERFORMANCE_SCHEMA_DB(_dbname)) #define qFatal(...) \ do { \ diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b01a871702..5c4bcf7946 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -49,6 +49,48 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC return TSDB_CODE_SUCCESS; } +static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { + int32_t code = 0; + SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); + if (NULL == vgInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + vgInfo->vgVersion = rsp->vgVersion; + vgInfo->stateTs = rsp->stateTs; + vgInfo->hashMethod = rsp->hashMethod; + vgInfo->hashPrefix = rsp->hashPrefix; + vgInfo->hashSuffix = rsp->hashSuffix; + vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == vgInfo->vgHash) { + taosMemoryFree(vgInfo); + tscError("hash init[%d] failed", rsp->vgNum); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _return; + } + + for (int32_t j = 0; j < rsp->vgNum; ++j) { + SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); + if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { + tscError("hash push failed, errno:%d", errno); + taosHashCleanup(vgInfo->vgHash); + taosMemoryFree(vgInfo); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _return; + } + } + +_return: + if (code) { + taosHashCleanup(vgInfo->vgHash); + taosMemoryFreeClear(vgInfo); + } + + *pInfo = vgInfo; + return code; +} + static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -67,37 +109,22 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); } else { - SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); - if (NULL == vgInfo) { - code = TSDB_CODE_OUT_OF_MEMORY; + SDBVgInfo *vgInfo = NULL; + code = hbGenerateVgInfoFromRsp(&vgInfo, rsp); + if (TSDB_CODE_SUCCESS != code) { goto _return; } - vgInfo->vgVersion = rsp->vgVersion; - vgInfo->stateTs = rsp->stateTs; - vgInfo->hashMethod = rsp->hashMethod; - vgInfo->hashPrefix = rsp->hashPrefix; - vgInfo->hashSuffix = rsp->hashSuffix; - vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == vgInfo->vgHash) { - taosMemoryFree(vgInfo); - tscError("hash init[%d] failed", rsp->vgNum); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _return; - } - - for (int32_t j = 0; j < rsp->vgNum; ++j) { - SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); - if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { - tscError("hash push failed, errno:%d", errno); - taosHashCleanup(vgInfo->vgHash); - taosMemoryFree(vgInfo); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _return; - } - } - catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, vgInfo); + + if (IS_SYS_DBNAME(rsp->db)) { + code = hbGenerateVgInfoFromRsp(&vgInfo, rsp); + if (TSDB_CODE_SUCCESS != code) { + goto _return; + } + + catalogUpdateDBVgInfo(pCatalog, (rsp->db[0] == 'i') ? TSDB_PERFORMANCE_SCHEMA_DB : TSDB_INFORMATION_SCHEMA_DB, rsp->uid, vgInfo); + } } if (code) { @@ -492,6 +519,9 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl for (int32_t i = 0; i < dbNum; ++i) { SDbVgVersion *db = &dbs[i]; + tscDebug("the %dth expired dbFName:%s, dbId:%" PRId64 ", vgVersion:%d, numOfTable:%d, startTs:%" PRId64, + i, db->dbFName, db->dbId, db->vgVersion, db->numOfTable, db->stateTs); + db->dbId = htobe64(db->dbId); db->vgVersion = htonl(db->vgVersion); db->numOfTable = htonl(db->numOfTable); diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 0596241ce1..54bfab1ebc 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -195,6 +195,7 @@ class TDTestCase: tdSql.checkData(1, 1, 1) tdSql.checkData(1, 2, '{"k1":1,"k2":"hello"}') + time.sleep(10) tdSql.query("select * from information_schema.ins_tables where table_name = 'stt4'") uid1 = tdSql.getData(0, 5) uid2 = tdSql.getData(1, 5) From f6827c3f060242c809233de3d66881a0e7389cd7 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 24 Feb 2023 13:53:48 +0800 Subject: [PATCH 6/6] fix: systable header file compile issue --- include/common/systable.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/common/systable.h b/include/common/systable.h index 6f65c1e8b8..cfc0af0172 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -12,6 +12,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +#ifndef TDENGINE_SYSTABLE_H +#define TDENGINE_SYSTABLE_H #ifdef __cplusplus extern "C" { @@ -19,9 +22,6 @@ extern "C" { #include "os.h" -#ifndef TDENGINE_SYSTABLE_H -#define TDENGINE_SYSTABLE_H - #define TSDB_INFORMATION_SCHEMA_DB "information_schema" #define TSDB_INS_TABLE_DNODES "ins_dnodes" #define TSDB_INS_TABLE_MNODES "ins_mnodes"