diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md
index fd6d3a2930..5e4eadcceb 100755
--- a/docs/en/14-reference/12-config/index.md
+++ b/docs/en/14-reference/12-config/index.md
@@ -421,7 +421,7 @@ The charset that takes effect is UTF-8.
| Applicable | Server Only |
| Meaning | Maximum number of vnodes per dnode |
| Value Range | 0-4096 |
-| Default Value | 2x the CPU cores |
+| Default Value | 2x the CPU cores plus 5 |
## Performance Tuning
diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md
index d4487e6148..f295e57bb5 100644
--- a/docs/en/28-releases/01-tdengine.md
+++ b/docs/en/28-releases/01-tdengine.md
@@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t
import Release from "/components/ReleaseV3";
+## 3.3.0.3
+
+
+
## 3.3.0.0
diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md
index f0347f424c..6fce985927 100755
--- a/docs/zh/14-reference/12-config/index.md
+++ b/docs/zh/14-reference/12-config/index.md
@@ -419,7 +419,7 @@ charset 的有效值是 UTF-8。
| 适用范围 | 仅服务端适用 |
| 含义 | dnode 支持的最大 vnode 数目 |
| 取值范围 | 0-4096 |
-| 缺省值 | CPU 核数的 2 倍 |
+| 缺省值 | CPU 核数的 2 倍 + 5 |
## 性能调优
diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md
index 93b24c9b69..f69e1fd4a8 100644
--- a/docs/zh/28-releases/01-tdengine.md
+++ b/docs/zh/28-releases/01-tdengine.md
@@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
import Release from "/components/ReleaseV3";
+## 3.3.0.3
+
+
+
## 3.3.0.0
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 32cd7bb2ab..197fa125f5 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -274,7 +274,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf, c
int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema, int64_t uid, int32_t vgId,
tb_uid_t suid);
-bool alreadyAddGroupId(char* ctbName);
+bool alreadyAddGroupId(char* ctbName, int64_t groupId);
bool isAutoTableName(char* ctbName);
void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId);
char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 17ed732af8..95b7591263 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -257,8 +257,10 @@ extern bool tsExperimental;
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
+int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd,
+ const char *envFile, char *apolloUrl, SArray *pArgs);
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
- bool tsc, bool isDumpCfg);
+ bool tsc);
void taosCleanupCfg();
int32_t taosCfgDynamicOptions(SConfig *pCfg, const char *name, bool forServer);
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 3ed6b40d4d..be2f443140 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2358,20 +2358,24 @@ void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp* pRsp);
typedef struct {
int32_t dnodeId;
char* token;
-} SMArbUpdateGroupReqMember;
+} SMArbUpdateGroupMember;
typedef struct {
- int32_t vgId;
- int64_t dbUid;
- SMArbUpdateGroupReqMember members[2];
- int8_t isSync;
- SMArbUpdateGroupReqMember assignedLeader;
- int64_t version;
-} SMArbUpdateGroupReq;
+ int32_t vgId;
+ int64_t dbUid;
+ SMArbUpdateGroupMember members[2];
+ int8_t isSync;
+ SMArbUpdateGroupMember assignedLeader;
+ int64_t version;
+} SMArbUpdateGroup;
-int32_t tSerializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq);
-int32_t tDeserializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq);
-void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq* pReq);
+typedef struct {
+ SArray* updateArray; // SMArbUpdateGroup
+} SMArbUpdateGroupBatchReq;
+
+int32_t tSerializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq);
+int32_t tDeserializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq);
+void tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq* pReq);
typedef struct {
char queryStrId[TSDB_QUERY_ID_LEN];
diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index a5a3bd5ee0..6f15d7df70 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -388,7 +388,8 @@
TD_NEW_MSG_SEG(TDMT_MND_ARB_MSG) //9 << 8
TD_DEF_MSG_TYPE(TDMT_MND_ARB_HEARTBEAT_TIMER, "mnd-arb-hb-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_CHECK_SYNC_TIMER, "mnd-arb-check-sync-tmr", NULL, NULL)
- TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL)
+ TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) // no longer used
+ TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP_BATCH, "mnd-arb-update-group-batch", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_ARB_MAX_MSG, "mnd-arb-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_ARB_MSG)
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index dafdac9649..9ae75bade2 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -327,7 +327,6 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) //
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A)
#define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B)
-#define TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x039C)
// mnode-node
#define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0)
@@ -612,6 +611,16 @@ int32_t* taosGetErrno();
#define TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0821)
#define TSDB_CODE_GRANT_DUPLICATED_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x0822)
#define TSDB_CODE_GRANT_VIEW_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0823)
+#define TSDB_CODE_GRANT_BASIC_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0824)
+#define TSDB_CODE_GRANT_STREAM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0825)
+#define TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0826)
+#define TSDB_CODE_GRANT_VIEW_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0827)
+#define TSDB_CODE_GRANT_AUDIT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0828)
+#define TSDB_CODE_GRANT_CSV_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0829)
+#define TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082A)
+#define TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082B)
+#define TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082C)
+#define TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082D)
// sync
// #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 036cf0420a..439103e5c4 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -725,7 +725,7 @@ void taos_init_imp(void) {
return;
}
- if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1, true) != 0) {
+ if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {
tscInitRes = -1;
return;
}
diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c
index 7d30a19140..1b6cb8fd22 100644
--- a/source/client/src/clientHb.c
+++ b/source/client/src/clientHb.c
@@ -92,6 +92,7 @@ static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *bat
}
if (!pRsp) {
releaseTscObj(pReq->connKey.tscRid);
+ taosHashCancelIterate(hbMgr->activeInfo, pReq);
break;
}
}
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 907a6ebe3c..3141e21f26 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1805,14 +1805,15 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
}
for (int j = 0; j < numOfVg; j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
- if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 10ms
+ int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs;
+ if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms
tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId,
tmq->epoch, pVg->vgId);
continue;
}
- if (tmq->replayEnable &&
- taosGetTimestampMs() - pVg->blockReceiveTs < pVg->blockSleepForReplay) { // less than 10ms
+ elapsed = taosGetTimestampMs() - pVg->blockReceiveTs;
+ if (tmq->replayEnable && elapsed < pVg->blockSleepForReplay && elapsed >= 0) {
tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay",
tmq->consumerId, tmq->epoch, pVg->vgId, pVg->blockSleepForReplay);
continue;
@@ -2127,7 +2128,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
if (timeout >= 0) {
int64_t currentTime = taosGetTimestampMs();
int64_t elapsedTime = currentTime - startTime;
- if (elapsedTime > timeout) {
+ if (elapsedTime > timeout || elapsedTime < 0) {
tscDebug("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64,
tmq->consumerId, tmq->epoch, startTime, currentTime);
return NULL;
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 1c0b696aaa..ac4811fb1b 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -2435,18 +2435,13 @@ void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId
// the total length is fixed to be 34 bytes.
bool isAutoTableName(char* ctbName) { return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_'); }
-bool alreadyAddGroupId(char* ctbName) {
- size_t len = strlen(ctbName);
- if (len == 0) return false;
- size_t _location = len - 1;
- while (_location > 0) {
- if (ctbName[_location] < '0' || ctbName[_location] > '9') {
- break;
- }
- _location--;
- }
-
- return ctbName[_location] == '_' && len - 1 - _location >= 15; // 15 means the min length of groupid
+bool alreadyAddGroupId(char* ctbName, int64_t groupId) {
+ char tmp[64] = {0};
+ snprintf(tmp, sizeof(tmp), "%" PRIu64, groupId);
+ size_t len1 = strlen(ctbName);
+ size_t len2 = strlen(tmp);
+ if (len1 < len2) return false;
+ return memcmp(ctbName + len1 - len2, tmp, len2) == 0;
}
char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 650a639964..691eccd174 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -624,7 +624,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
- tsNumOfSupportVnodes = tsNumOfCores * 2;
+ tsNumOfSupportVnodes = tsNumOfCores * 2 + 5;
tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1;
@@ -878,7 +878,7 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
pItem = cfgGetItem(tsCfg, "supportVnodes");
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
- tsNumOfSupportVnodes = numOfCores * 2;
+ tsNumOfSupportVnodes = numOfCores * 2 + 5;
tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
pItem->i32 = tsNumOfSupportVnodes;
pItem->stype = stype;
@@ -1377,6 +1377,35 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
return 0;
}
+int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd,
+ const char *envFile, char *apolloUrl, SArray *pArgs) {
+ if (tsCfg == NULL) osDefaultInit();
+
+ SConfig *pCfg = cfgInit();
+ if (pCfg == NULL) return -1;
+
+ if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+ if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+
+ if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
+ printf("failed to load cfg since %s", terrstr());
+ cfgCleanup(pCfg);
+ return -1;
+ }
+
+ if (cfgLoadFromArray(pCfg, pArgs) != 0) {
+ printf("failed to load cfg from array since %s", terrstr());
+ cfgCleanup(pCfg);
+ return -1;
+ }
+
+ tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX);
+ dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32;
+
+ cfgCleanup(pCfg);
+ return 0;
+}
+
static int32_t taosCheckGlobalCfg() {
uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn);
if (ipv4 == 0xffffffff) {
@@ -1394,7 +1423,7 @@ static int32_t taosCheckGlobalCfg() {
}
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
- bool tsc, bool isDumpCfg) {
+ bool tsc) {
if (tsCfg != NULL) return 0;
tsCfg = cfgInit();
@@ -1441,7 +1470,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
- if(isDumpCfg) cfgDumpCfg(tsCfg, tsc, false);
+ cfgDumpCfg(tsCfg, tsc, false);
if (taosCheckGlobalCfg() != 0) {
return -1;
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index b7d1417451..ef37a41fcf 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -6437,21 +6437,28 @@ void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp *pRsp) {
taosMemoryFreeClear(pRsp->memberToken);
}
-int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) {
+int32_t tSerializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
- if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
- for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
- if (tEncodeI32(&encoder, pReq->members[i].dnodeId) < 0) return -1;
- if (tEncodeCStr(&encoder, pReq->members[i].token) < 0) return -1;
+
+ int32_t sz = taosArrayGetSize(pReq->updateArray);
+ if (tEncodeI32(&encoder, sz) < 0) return -1;
+
+ for (int32_t i = 0; i < sz; i++) {
+ SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i);
+ if (tEncodeI32(&encoder, pGroup->vgId) < 0) return -1;
+ if (tEncodeI64(&encoder, pGroup->dbUid) < 0) return -1;
+ for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+ if (tEncodeI32(&encoder, pGroup->members[i].dnodeId) < 0) return -1;
+ if (tEncodeCStr(&encoder, pGroup->members[i].token) < 0) return -1;
+ }
+ if (tEncodeI8(&encoder, pGroup->isSync) < 0) return -1;
+ if (tEncodeI32(&encoder, pGroup->assignedLeader.dnodeId) < 0) return -1;
+ if (tEncodeCStr(&encoder, pGroup->assignedLeader.token) < 0) return -1;
+ if (tEncodeI64(&encoder, pGroup->version) < 0) return -1;
}
- if (tEncodeI8(&encoder, pReq->isSync) < 0) return -1;
- if (tEncodeI32(&encoder, pReq->assignedLeader.dnodeId) < 0) return -1;
- if (tEncodeCStr(&encoder, pReq->assignedLeader.token) < 0) return -1;
- if (tEncodeI64(&encoder, pReq->version) < 0) return -1;
tEndEncode(&encoder);
@@ -6460,23 +6467,34 @@ int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGrou
return tlen;
}
-int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) {
+int32_t tDeserializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
- if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
- for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
- if (tDecodeI32(&decoder, &pReq->members[i].dnodeId) < 0) return -1;
- pReq->members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
- if (tDecodeCStrTo(&decoder, pReq->members[i].token) < 0) return -1;
+ int32_t sz = 0;
+ if (tDecodeI32(&decoder, &sz) < 0) return -1;
+
+ SArray *updateArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup));
+ if (!updateArray) return -1;
+
+ for (int32_t i = 0; i < sz; i++) {
+ SMArbUpdateGroup group = {0};
+ if (tDecodeI32(&decoder, &group.vgId) < 0) return -1;
+ if (tDecodeI64(&decoder, &group.dbUid) < 0) return -1;
+ for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+ if (tDecodeI32(&decoder, &group.members[i].dnodeId) < 0) return -1;
+ group.members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
+ if (tDecodeCStrTo(&decoder, group.members[i].token) < 0) return -1;
+ }
+ if (tDecodeI8(&decoder, &group.isSync) < 0) return -1;
+ if (tDecodeI32(&decoder, &group.assignedLeader.dnodeId) < 0) return -1;
+ group.assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
+ if (tDecodeCStrTo(&decoder, group.assignedLeader.token) < 0) return -1;
+ if (tDecodeI64(&decoder, &group.version) < 0) return -1;
+ taosArrayPush(updateArray, &group);
}
- if (tDecodeI8(&decoder, &pReq->isSync) < 0) return -1;
- if (tDecodeI32(&decoder, &pReq->assignedLeader.dnodeId) < 0) return -1;
- pReq->assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
- if (tDecodeCStrTo(&decoder, pReq->assignedLeader.token) < 0) return -1;
- if (tDecodeI64(&decoder, &pReq->version) < 0) return -1;
+ pReq->updateArray = updateArray;
tEndDecode(&decoder);
@@ -6484,14 +6502,20 @@ int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGr
return 0;
}
-void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq *pReq) {
- if (NULL == pReq) {
+void tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq *pReq) {
+ if (NULL == pReq || NULL == pReq->updateArray) {
return;
}
- for (int i = 0; i < 2; i++) {
- taosMemoryFreeClear(pReq->members[i].token);
+
+ int32_t sz = taosArrayGetSize(pReq->updateArray);
+ for (int32_t i = 0; i < sz; i++) {
+ SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i);
+ for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+ taosMemoryFreeClear(pGroup->members[i].token);
+ }
+ taosMemoryFreeClear(pGroup->assignedLeader.token);
}
- taosMemoryFreeClear(pReq->assignedLeader.token);
+ taosArrayDestroy(pReq->updateArray);
}
// int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {
diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp
index 8e0e50165f..d30b26e564 100644
--- a/source/common/test/commonTests.cpp
+++ b/source/common/test/commonTests.cpp
@@ -692,4 +692,38 @@ TEST(timeTest, epSet) {
ASSERT_EQ(ep.numOfEps, 1);
}
}
+
+// Define test cases
+TEST(AlreadyAddGroupIdTest, GroupIdAdded) {
+ // Test case 1: Group ID has been added
+ char ctbName[64] = "abc123";
+ int64_t groupId = 123;
+ bool result = alreadyAddGroupId(ctbName, groupId);
+ EXPECT_TRUE(result);
+}
+
+TEST(AlreadyAddGroupIdTest, GroupIdNotAdded) {
+ // Test case 2: Group ID has not been added
+ char ctbName[64] = "abc456";
+ int64_t groupId = 123;
+ bool result = alreadyAddGroupId(ctbName, groupId);
+ EXPECT_FALSE(result);
+}
+
+TEST(AlreadyAddGroupIdTest, GroupIdAddedAtTheEnd) {
+ // Test case 3: Group ID has been added at the end
+ char ctbName[64] = "xyz1";
+ int64_t groupId = 1;
+ bool result = alreadyAddGroupId(ctbName, groupId);
+ EXPECT_TRUE(result);
+}
+
+TEST(AlreadyAddGroupIdTest, GroupIdAddedWithDifferentLength) {
+ // Test case 4: Group ID has been added with different length
+ char ctbName[64] = "def";
+ int64_t groupId = 123456;
+ bool result = alreadyAddGroupId(ctbName, groupId);
+ EXPECT_FALSE(result);
+}
+
#pragma GCC diagnostic pop
diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c
index 6a68f580b2..cdae6a493c 100644
--- a/source/dnode/mgmt/exe/dmMain.c
+++ b/source/dnode/mgmt/exe/dmMain.c
@@ -205,11 +205,11 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
if(i < argc - 1) {
int32_t len = strlen(argv[++i]);
if (len < ENCRYPT_KEY_LEN_MIN) {
- printf("Error: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN);
+ printf("ERROR: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN);
return -1;
}
if (len > ENCRYPT_KEY_LEN) {
- printf("Error: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN);
+ printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN);
return -1;
}
tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
@@ -366,6 +366,22 @@ int mainWindows(int argc, char **argv) {
printf("memory dbg enabled\n");
}
#endif
+ if(global.generateCode) {
+ bool toLogFile = false;
+ if(taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs) != 0){
+ encryptError("failed to generate encrypt code since taosd is running, please stop it first");
+ return -1;
+ };
+
+ if(dmCheckRunning(tsDataDir) == NULL) {
+ encryptError("failed to generate encrypt code since taosd is running, please stop it first");
+ return -1;
+ }
+ int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile);
+ taosCloseLog();
+ taosCleanupArgs();
+ return ret;
+ }
if (dmInitLog() != 0) {
printf("failed to start since init log error\n");
@@ -375,28 +391,13 @@ int mainWindows(int argc, char **argv) {
dmPrintArgs(argc, argv);
- bool isDumpCfg = true;
- if(global.generateCode) {
- isDumpCfg = false;
- }
- if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0, isDumpCfg) != 0) {
+ if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
dError("failed to start since read config error");
taosCloseLog();
taosCleanupArgs();
return -1;
}
- if(global.generateCode) {
- if(dmCheckRunning(tsDataDir) == NULL) {
- dError("failed to generate encrypt code since taosd is running, please stop it first");
- return -1;
- }
- int ret = dmUpdateEncryptKey(global.encryptKey);
- taosCloseLog();
- taosCleanupArgs();
- return ret;
- }
-
if(dmGetEncryptKey() != 0){
dError("failed to start since failed to get encrypt key");
taosCloseLog();
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
index 3ec080fb21..800f7f7864 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
@@ -220,7 +220,7 @@ int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
goto _exit;
}
- code = dmUpdateEncryptKey(cfgReq.value);
+ code = dmUpdateEncryptKey(cfgReq.value, true);
if (code == 0) {
tsEncryptionKeyChksum = taosCalcChecksum(0, cfgReq.value, strlen(cfgReq.value));
tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED;
diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h
index 9fdd5e50ed..aea3286d76 100644
--- a/source/dnode/mgmt/node_util/inc/dmUtil.h
+++ b/source/dnode/mgmt/node_util/inc/dmUtil.h
@@ -53,6 +53,36 @@ extern "C" {
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
+#define encryptDebug(...) { \
+ if (toLogFile) { \
+ if (dDebugFlag & DEBUG_DEBUG) {taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__);} \
+ } else { \
+ /*if (dDebugFlag & DEBUG_DEBUG) {taosPrintLog("DND ", DEBUG_SCREEN, dDebugFlag, __VA_ARGS__);}*/ \
+ if (dDebugFlag & DEBUG_DEBUG) {printf(__VA_ARGS__); printf("\n");} \
+ } \
+}
+
+#define encryptInfo(...) { \
+ if (toLogFile) { \
+ taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); \
+ } else { \
+ /*if (dDebugFlag & DEBUG_DEBUG) {taosPrintLog("DND ", DEBUG_SCREEN, dDebugFlag, __VA_ARGS__);}*/ \
+ printf(__VA_ARGS__); \
+ printf("\n"); \
+ } \
+}
+
+#define encryptError(...) { \
+ if (toLogFile) { \
+ taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
+ }\
+ else{ \
+ /*taosPrintLog("DND ", DEBUG_SCREEN, 255, __VA_ARGS__); */\
+ printf("ERROR: " __VA_ARGS__); \
+ printf("\n"); \
+ }\
+}
+
#define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", gtid:%s", __VA_ARGS__, buf);}}
#define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", gtid:%s", __VA_ARGS__, buf);}}
@@ -194,7 +224,7 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
bool dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port);
void dmRemoveDnodePairs(SDnodeData *pData);
void dmGetDnodeEp(void *pData, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
-int32_t dmUpdateEncryptKey(char *key);
+int32_t dmUpdateEncryptKey(char *key, bool toLogFile);
int32_t dmGetEncryptKey();
#ifdef __cplusplus
}
diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c
index e2c52cfb7e..752abb83a2 100644
--- a/source/dnode/mgmt/node_util/src/dmFile.c
+++ b/source/dnode/mgmt/node_util/src/dmFile.c
@@ -186,7 +186,7 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode);
-static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key){
+static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool toLogFile){
TdFilePtr pFile = NULL;
char *result = NULL;
int32_t code = -1;
@@ -211,7 +211,8 @@ static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key){
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
- dInfo("succeed to write checkCode file:%s", realfile);
+ encryptDebug("succeed to write checkCode file:%s", realfile);
+
code = 0;
_OVER:
if(pFile != NULL) taosCloseFile(&pFile);
@@ -220,7 +221,7 @@ _OVER:
return code;
}
-static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode){
+static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode, bool toLogFile){
TdFilePtr pFile = NULL;
int32_t code = -1;
@@ -234,7 +235,7 @@ static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptC
taosCloseFile(&pFile);
if (taosRenameFile(file, realfile) != 0) goto _OVER;
- dInfo("succeed to write encryptCode file:%s", realfile);
+ encryptDebug("succeed to write encryptCode file:%s", realfile);
code = 0;
_OVER:
@@ -243,7 +244,7 @@ _OVER:
return code;
}
-static int32_t dmCompareEncryptKey(char* file, char* key){
+static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){
char *content = NULL;
int64_t size = 0;
TdFilePtr pFile = NULL;
@@ -253,13 +254,13 @@ static int32_t dmCompareEncryptKey(char* file, char* key){
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
- dError("failed to open dnode file:%s since %s", file, terrstr());
+ encryptError("failed to open dnode file:%s since %s", file, terrstr());
goto _OVER;
}
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
- dError("failed to fstat dnode file:%s since %s", file, terrstr());
+ encryptError("failed to fstat dnode file:%s since %s", file, terrstr());
goto _OVER;
}
@@ -271,11 +272,11 @@ static int32_t dmCompareEncryptKey(char* file, char* key){
if (taosReadFile(pFile, content, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
- dError("failed to read dnode file:%s since %s", file, terrstr());
+ encryptError("failed to read dnode file:%s since %s", file, terrstr());
goto _OVER;
}
- dInfo("succeed to read checkCode file:%s", file);
+ encryptDebug("succeed to read checkCode file:%s", file);
int len = ENCRYPTED_LEN(size);
result = taosMemoryMalloc(len);
@@ -290,11 +291,11 @@ static int32_t dmCompareEncryptKey(char* file, char* key){
if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) {
terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED;
- dError("failed to compare decrypted result");
+ encryptError("failed to compare decrypted result");
goto _OVER;
}
- dInfo("succeed to compare checkCode file:%s", file);
+ encryptDebug("succeed to compare checkCode file:%s", file);
code = 0;
_OVER:
if(result != NULL) taosMemoryFree(result);
@@ -304,7 +305,7 @@ _OVER:
return code;
}
-int32_t dmUpdateEncryptKey(char *key) {
+int32_t dmUpdateEncryptKey(char *key, bool toLogFile) {
#ifdef TD_ENTERPRISE
int32_t code = -1;
char *machineId = NULL;
@@ -328,12 +329,12 @@ int32_t dmUpdateEncryptKey(char *key) {
if (taosMkDir(folder) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
- dError("failed to create dir:%s since %s", folder, terrstr());
+ encryptError("failed to create dir:%s since %s", folder, terrstr());
goto _OVER;
}
if(taosCheckExistFile(realCheckFile)){
- if(dmCompareEncryptKey(realCheckFile, key) != 0){
+ if(dmCompareEncryptKey(realCheckFile, key, toLogFile) != 0){
goto _OVER;
}
}
@@ -347,21 +348,23 @@ int32_t dmUpdateEncryptKey(char *key) {
goto _OVER;
}
- if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode) != 0){
+ if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile) != 0){
goto _OVER;
}
- if(dmWriteCheckCodeFile(checkFile, realCheckFile, key) != 0){
+ if(dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile) != 0){
goto _OVER;
}
+ encryptInfo("Succeed to update encrypt key\n");
+
code = 0;
_OVER:
taosMemoryFree(encryptCode);
taosMemoryFree(machineId);
if (code != 0) {
if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno);
- dError("failed to update encrypt key since %s", terrstr());
+ encryptError("failed to update encrypt key since %s", terrstr());
}
return code;
#else
@@ -453,7 +456,7 @@ int32_t dmGetEncryptKey(){
goto _OVER;
}
- if(dmCompareEncryptKey(checkFile, encryptKey) != 0){
+ if(dmCompareEncryptKey(checkFile, encryptKey, true) != 0){
goto _OVER;
}
diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h
index 68c55e235f..0c40622d08 100644
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@@ -180,7 +180,7 @@ typedef struct {
tmsg_t originRpcType;
char dbname[TSDB_TABLE_FNAME_LEN];
char stbname[TSDB_TABLE_FNAME_LEN];
- int32_t arbGroupId;
+ SHashObj* arbGroupIds;
int32_t startFunc;
int32_t stopFunc;
int32_t paramLen;
diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h
index 8c9ca87fb1..8008eb76e7 100644
--- a/source/dnode/mnode/impl/inc/mndTrans.h
+++ b/source/dnode/mnode/impl/inc/mndTrans.h
@@ -78,7 +78,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen);
void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname);
-void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId);
+void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId);
void mndTransSetSerial(STrans *pTrans);
void mndTransSetParallel(STrans *pTrans);
void mndTransSetChangeless(STrans *pTrans);
diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c
index d0a86bdde7..50338fe889 100644
--- a/source/dnode/mnode/impl/src/mndArbGroup.c
+++ b/source/dnode/mnode/impl/src/mndArbGroup.c
@@ -39,10 +39,11 @@ static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup);
static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew);
static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup);
+static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray);
static int32_t mndProcessArbHbTimer(SRpcMsg *pReq);
static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq);
-static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq);
+static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq);
static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp);
static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp);
static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp);
@@ -68,7 +69,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_ARB_HEARTBEAT_TIMER, mndProcessArbHbTimer);
mndSetMsgHandle(pMnode, TDMT_MND_ARB_CHECK_SYNC_TIMER, mndProcessArbCheckSyncTimer);
- mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP, mndProcessArbUpdateGroupReq);
+ mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP_BATCH, mndProcessArbUpdateGroupBatchReq);
mndSetMsgHandle(pMnode, TDMT_VND_ARB_HEARTBEAT_RSP, mndProcessArbHbRsp);
mndSetMsgHandle(pMnode, TDMT_VND_ARB_CHECK_SYNC_RSP, mndProcessArbCheckSyncRsp);
mndSetMsgHandle(pMnode, TDMT_SYNC_SET_ASSIGNED_LEADER_RSP, mndProcessArbSetAssignedLeaderRsp);
@@ -81,9 +82,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
return sdbSetTable(pMnode->pSdb, table);
}
-void mndCleanupArbGroup(SMnode *pMnode) {
- taosHashCleanup(arbUpdateHash);
-}
+void mndCleanupArbGroup(SMnode *pMnode) { taosHashCleanup(arbUpdateHash); }
SArbGroup *mndAcquireArbGroup(SMnode *pMnode, int32_t vgId) {
SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId);
@@ -541,6 +540,8 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
return -1;
}
+ SArray *pUpdateArray = taosArrayInit(16, sizeof(SArbGroup));
+
while (1) {
pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup);
if (pIter == NULL) break;
@@ -612,40 +613,27 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
SArbGroup newGroup = {0};
mndArbGroupDupObj(&arbGroupDup, &newGroup);
mndArbGroupSetAssignedLeader(&newGroup, candidateIndex);
- if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) {
- mError("vgId:%d, arb failed to pullup set assigned leader to dnodeId:%d, since %s", vgId, pMember->info.dnodeId,
- terrstr());
- sdbRelease(pSdb, pArbGroup);
- return -1;
- }
-
- mInfo("vgId:%d, arb pull up set assigned leader to dnodeId:%d", vgId, pMember->info.dnodeId);
+ taosArrayPush(pUpdateArray, &newGroup);
sdbRelease(pSdb, pArbGroup);
}
+ (void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray);
+
+ taosArrayDestroy(pUpdateArray);
return 0;
}
-static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup) {
- SMArbUpdateGroupReq req = {0};
- req.vgId = pNewGroup->vgId;
- req.dbUid = pNewGroup->dbUid;
- for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
- req.members[i].dnodeId = pNewGroup->members[i].info.dnodeId;
- req.members[i].token = pNewGroup->members[i].state.token;
- }
- req.isSync = pNewGroup->isSync;
- req.assignedLeader.dnodeId = pNewGroup->assignedLeader.dnodeId;
- req.assignedLeader.token = pNewGroup->assignedLeader.token;
- req.version = pNewGroup->version;
+static void *mndBuildArbUpdateGroupBatchReq(int32_t *pContLen, SArray *updateArray) {
+ SMArbUpdateGroupBatchReq req = {0};
+ req.updateArray = updateArray;
- int32_t contLen = tSerializeSMArbUpdateGroupReq(NULL, 0, &req);
+ int32_t contLen = tSerializeSMArbUpdateGroupBatchReq(NULL, 0, &req);
if (contLen <= 0) return NULL;
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) return NULL;
- if (tSerializeSMArbUpdateGroupReq(pHead, contLen, &req) <= 0) {
+ if (tSerializeSMArbUpdateGroupBatchReq(pHead, contLen, &req) <= 0) {
rpcFreeCont(pHead);
return NULL;
}
@@ -653,60 +641,172 @@ static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup)
return pHead;
}
+static void mndInitArbUpdateGroup(SArbGroup *pGroup, SMArbUpdateGroup *outGroup) {
+ outGroup->vgId = pGroup->vgId;
+ outGroup->dbUid = pGroup->dbUid;
+ for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+ outGroup->members[i].dnodeId = pGroup->members[i].info.dnodeId;
+ outGroup->members[i].token = pGroup->members[i].state.token; // just copy the pointer
+ }
+ outGroup->isSync = pGroup->isSync;
+ outGroup->assignedLeader.dnodeId = pGroup->assignedLeader.dnodeId;
+ outGroup->assignedLeader.token = pGroup->assignedLeader.token; // just copy the pointer
+ outGroup->version = pGroup->version;
+}
+
static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup) {
if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) {
mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId);
return 0;
}
- int32_t contLen = 0;
- void *pHead = mndBuildArbUpdateGroupReq(&contLen, pNewGroup);
- if (!pHead) {
- mError("vgId:%d, failed to build arb-update-group request", pNewGroup->vgId);
- return -1;
- }
- SRpcMsg rpcMsg = {.msgType = TDMT_MND_ARB_UPDATE_GROUP, .pCont = pHead, .contLen = contLen, .info.noResp = true};
+ int32_t ret = -1;
- int32_t ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
- if (ret == 0) {
- taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
+ SMArbUpdateGroup newGroup = {0};
+ mndInitArbUpdateGroup(pNewGroup, &newGroup);
+
+ SArray *pArray = taosArrayInit(1, sizeof(SMArbUpdateGroup));
+ taosArrayPush(pArray, &newGroup);
+
+ int32_t contLen = 0;
+ void *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray);
+ if (!pHead) {
+ mError("failed to build arb-update-group request");
+ goto _OVER;
}
+
+ SRpcMsg rpcMsg = {
+ .msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true};
+ ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+ if (ret != 0) goto _OVER;
+
+ taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
+
+_OVER:
+ taosArrayDestroy(pArray);
return ret;
}
-static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq) {
- int ret = 0;
+static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray) {
+ int32_t ret = -1;
- SMArbUpdateGroupReq req = {0};
- tDeserializeSMArbUpdateGroupReq(pReq->pCont, pReq->contLen, &req);
+ size_t sz = taosArrayGetSize(newGroupArray);
+ SArray *pArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup));
+ for (size_t i = 0; i < sz; i++) {
+ SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i);
+ if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) {
+ mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId);
+ continue;
+ }
- SArbGroup newGroup = {0};
- newGroup.vgId = req.vgId;
- newGroup.dbUid = req.dbUid;
- for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
- newGroup.members[i].info.dnodeId = req.members[i].dnodeId;
- memcpy(newGroup.members[i].state.token, req.members[i].token, TSDB_ARB_TOKEN_SIZE);
+ SMArbUpdateGroup newGroup = {0};
+ mndInitArbUpdateGroup(pNewGroup, &newGroup);
+
+ taosArrayPush(pArray, &newGroup);
+ taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
}
- newGroup.isSync = req.isSync;
- newGroup.assignedLeader.dnodeId = req.assignedLeader.dnodeId;
- memcpy(newGroup.assignedLeader.token, req.assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
- newGroup.version = req.version;
-
- SMnode *pMnode = pReq->info.node;
- SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId);
- if (!pOldGroup) {
- mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId);
- return 0;
- }
- sdbRelease(pMnode->pSdb, pOldGroup);
-
- if (mndArbGroupUpdateTrans(pMnode, &newGroup) != 0) {
- mError("vgId:%d, arb failed to update arbgroup, since %s", newGroup.vgId, terrstr());
- ret = -1;
+ if (taosArrayGetSize(pArray) == 0) {
+ ret = 0;
+ goto _OVER;
}
- tFreeSMArbUpdateGroupReq(&req);
+ int32_t contLen = 0;
+ void *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray);
+ if (!pHead) {
+ mError("failed to build arb-update-group request");
+ goto _OVER;
+ }
+
+ SRpcMsg rpcMsg = {
+ .msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true};
+ ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+
+_OVER:
+ taosArrayDestroy(pArray);
+
+ if (ret != 0) {
+ for (size_t i = 0; i < sz; i++) {
+ SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i);
+ taosHashRemove(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId));
+ }
+ }
+
+ return ret;
+}
+
+static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
+ int ret = -1;
+ size_t sz = 0;
+
+ SMArbUpdateGroupBatchReq req = {0};
+ if (tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req) != 0) {
+ mError("arb failed to decode arb-update-group request");
+ return -1;
+ }
+
+ SMnode *pMnode = pReq->info.node;
+ STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup");
+ if (pTrans == NULL) {
+ mError("failed to update arbgroup in create trans, since %s", terrstr());
+ goto _OVER;
+ }
+
+ sz = taosArrayGetSize(req.updateArray);
+ for (size_t i = 0; i < sz; i++) {
+ SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i);
+ SArbGroup newGroup = {0};
+ newGroup.vgId = pUpdateGroup->vgId;
+ newGroup.dbUid = pUpdateGroup->dbUid;
+ for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+ newGroup.members[i].info.dnodeId = pUpdateGroup->members[i].dnodeId;
+ memcpy(newGroup.members[i].state.token, pUpdateGroup->members[i].token, TSDB_ARB_TOKEN_SIZE);
+ }
+
+ newGroup.isSync = pUpdateGroup->isSync;
+ newGroup.assignedLeader.dnodeId = pUpdateGroup->assignedLeader.dnodeId;
+ memcpy(newGroup.assignedLeader.token, pUpdateGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
+ newGroup.version = pUpdateGroup->version;
+
+ SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId);
+ if (!pOldGroup) {
+ mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId);
+ taosHashRemove(arbUpdateHash, &newGroup.vgId, sizeof(int32_t));
+ continue;
+ }
+
+ mndTransAddArbGroupId(pTrans, newGroup.vgId);
+
+ if (mndSetCreateArbGroupCommitLogs(pTrans, &newGroup) != 0) {
+ mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id,
+ terrstr());
+ goto _OVER;
+ }
+
+ mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s]",
+ pTrans->id, newGroup.vgId, newGroup.members[0].info.dnodeId, newGroup.members[0].state.token,
+ newGroup.members[1].info.dnodeId, newGroup.members[1].state.token, newGroup.isSync,
+ newGroup.assignedLeader.dnodeId, newGroup.assignedLeader.token);
+
+ sdbRelease(pMnode->pSdb, pOldGroup);
+ }
+
+ if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
+ if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
+
+ ret = 0;
+
+_OVER:
+ if (ret != 0) {
+ // failed to update arbgroup
+ for (size_t i = 0; i < sz; i++) {
+ SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i);
+ taosHashRemove(arbUpdateHash, &pUpdateGroup->vgId, sizeof(int32_t));
+ }
+ }
+
+ mndTransDrop(pTrans);
+ tFreeSMArbUpdateGroupBatchReq(&req);
return ret;
}
@@ -739,7 +839,7 @@ static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
pNew->members[1].info.dnodeId, pNew->members[1].state.token, pNew->isSync, pNew->assignedLeader.dnodeId,
pNew->assignedLeader.token);
- mndTransSetArbGroupId(pTrans, pNew->vgId);
+ mndTransAddArbGroupId(pTrans, pNew->vgId);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
ret = -1;
goto _OVER;
@@ -816,10 +916,10 @@ _OVER:
}
static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *memberArray) {
- int ret = 0;
int64_t nowMs = taosGetTimestampMs();
+ size_t size = taosArrayGetSize(memberArray);
+ SArray *pUpdateArray = taosArrayInit(size, sizeof(SArbGroup));
- size_t size = taosArrayGetSize(memberArray);
for (size_t i = 0; i < size; i++) {
SVArbHbRspMember *pRspMember = taosArrayGet(memberArray, i);
@@ -832,17 +932,16 @@ static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *me
bool updateToken = mndUpdateArbGroupByHeartBeat(pGroup, pRspMember, nowMs, dnodeId, &newGroup);
if (updateToken) {
- ret = mndPullupArbUpdateGroup(pMnode, &newGroup);
- if (ret != 0) {
- mInfo("failed to pullup update arb token, vgId:%d, since %s", pRspMember->vgId, terrstr());
- }
+ taosArrayPush(pUpdateArray, &newGroup);
}
sdbRelease(pMnode->pSdb, pGroup);
- if (ret != 0) break;
}
- return ret;
+ (void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray);
+
+ taosArrayDestroy(pUpdateArray);
+ return 0;
}
bool mndUpdateArbGroupByCheckSync(SArbGroup *pGroup, int32_t vgId, char *member0Token, char *member1Token,
@@ -900,6 +999,11 @@ static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token
}
static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) {
+ if (pRsp->contLen == 0) {
+ mDebug("arb hb-rsp contLen is 0");
+ return 0;
+ }
+
int32_t ret = -1;
SMnode *pMnode = pRsp->info.node;
@@ -914,6 +1018,7 @@ static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) {
SVArbHeartBeatRsp arbHbRsp = {0};
if (tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp) != 0) {
+ mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
@@ -934,6 +1039,11 @@ _OVER:
}
static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
+ if (pRsp->contLen == 0) {
+ mDebug("arb check-sync-rsp contLen is 0");
+ return 0;
+ }
+
int32_t ret = -1;
SMnode *pMnode = pRsp->info.node;
@@ -948,7 +1058,7 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
SVArbCheckSyncRsp syncRsp = {0};
if (tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp) != 0) {
- mInfo("arb sync check failed, since:%s", tstrerror(pRsp->code));
+ mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code));
if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) {
terrno = TSDB_CODE_SUCCESS;
return 0;
@@ -1008,6 +1118,11 @@ _OVER:
}
static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
+ if (pRsp->contLen == 0) {
+ mDebug("arb set-assigned-rsp contLen is 0");
+ return 0;
+ }
+
int32_t ret = -1;
SMnode *pMnode = pRsp->info.node;
@@ -1022,8 +1137,8 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
SVArbSetAssignedLeaderRsp setAssignedRsp = {0};
if (tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp) != 0) {
+ mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG;
- mInfo("arb set assigned failed, des failed since:%s", tstrerror(pRsp->code));
return -1;
}
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index f254fb16a5..ad596289de 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -821,8 +821,7 @@ static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) {
#ifdef TD_ENTERPRISE
if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit;
- if (grantCheck(TSDB_GRANT_DB_ENCRYPTION) != 0) {
- code = TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED;
+ if ((code = grantCheck(TSDB_GRANT_DB_ENCRYPTION)) != 0) {
goto _exit;
}
if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) {
@@ -1226,7 +1225,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
_OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
if (terrno != 0) code = terrno;
- mError("db:%s, failed to alter since %s", alterReq.db, terrstr());
+ mError("db:%s, failed to alter since %s", alterReq.db, tstrerror(code));
}
mndReleaseDb(pMnode, pDb);
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index fb15e4b857..44fe81ac09 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1847,8 +1847,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL;
- if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) {
- terrno = TSDB_CODE_GRANT_EXPIRED;
+ if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
return -1;
}
diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c
index a81a391c3d..9bd7b3b18f 100644
--- a/source/dnode/mnode/impl/src/mndStreamHb.c
+++ b/source/dnode/mnode/impl/src/mndStreamHb.c
@@ -225,7 +225,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SArray *pFailedTasks = NULL;
SArray *pOrphanTasks = NULL;
- if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) {
+ if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
if (suspendAllStreams(pMnode, &pReq->info) < 0) {
return -1;
}
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 84940e01d4..ecc163985c 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -26,7 +26,7 @@
#define TRANS_VER1_NUMBER 1
#define TRANS_VER2_NUMBER 2
#define TRANS_ARRAY_SIZE 8
-#define TRANS_RESERVE_SIZE 48
+#define TRANS_RESERVE_SIZE 44
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld);
@@ -196,10 +196,21 @@ SSdbRaw *mndTransEncode(STrans *pTrans) {
}
SDB_SET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER)
+
+ int32_t arbGroupNum = taosHashGetSize(pTrans->arbGroupIds);
+ SDB_SET_INT32(pRaw, dataPos, arbGroupNum, _OVER)
+ void *pIter = NULL;
+ pIter = taosHashIterate(pTrans->arbGroupIds, NULL);
+ while (pIter) {
+ int32_t arbGroupId = *(int32_t *)pIter;
+ SDB_SET_INT32(pRaw, dataPos, arbGroupId, _OVER)
+ pIter = taosHashIterate(pTrans->arbGroupIds, pIter);
+ }
+
SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
- terrno = 0;
+ terrno = 0;
_OVER:
if (terrno != 0) {
@@ -279,6 +290,7 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) {
int32_t undoActionNum = 0;
int32_t commitActionNum = 0;
int32_t dataPos = 0;
+ int32_t arbgroupIdNum = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@@ -350,6 +362,16 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) {
}
SDB_GET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER);
+
+ pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
+
+ SDB_GET_INT32(pRaw, dataPos, &arbgroupIdNum, _OVER)
+ for (int32_t i = 0; i < arbgroupIdNum; ++i) {
+ int32_t arbGroupId = 0;
+ SDB_GET_INT32(pRaw, dataPos, &arbGroupId, _OVER)
+ taosHashPut(pTrans->arbGroupIds, &arbGroupId, sizeof(int32_t), NULL, 0);
+ }
+
SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER)
terrno = 0;
@@ -462,6 +484,9 @@ void mndTransDropData(STrans *pTrans) {
mndTransDropActions(pTrans->commitActions);
pTrans->commitActions = NULL;
}
+ if (pTrans->arbGroupIds != NULL) {
+ taosHashCleanup(pTrans->arbGroupIds);
+ }
if (pTrans->pRpcArray != NULL) {
taosArrayDestroy(pTrans->pRpcArray);
pTrans->pRpcArray = NULL;
@@ -581,6 +606,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
+ pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo));
pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64();
taosInitRWLatch(&pTrans->lockRpcArray);
@@ -733,7 +759,9 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname)
}
}
-void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId) { pTrans->arbGroupId = groupId; }
+void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId) {
+ taosHashPut(pTrans->arbGroupIds, &groupId, sizeof(int32_t), NULL, 0);
+}
void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }
@@ -821,7 +849,16 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
if (pNew->conflict == TRN_CONFLICT_ARBGROUP) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) {
- if (pNew->arbGroupId == pTrans->arbGroupId) conflict = true;
+ void *pIter = taosHashIterate(pNew->arbGroupIds, NULL);
+ while (pIter != NULL) {
+ int32_t groupId = *(int32_t *)pIter;
+ if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) {
+ taosHashCancelIterate(pNew->arbGroupIds, pIter);
+ conflict = true;
+ break;
+ }
+ pIter = taosHashIterate(pNew->arbGroupIds, pIter);
+ }
}
}
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 04a621d4f5..c85316f810 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -922,9 +922,16 @@ typedef struct {
SColVal colVal;
} SLastCol;
+typedef struct {
+ int8_t lflag;
+ STsdbRowKey tsdbRowKey;
+ SColVal colVal;
+} SLastUpdateCtx;
+
int32_t tsdbOpenCache(STsdb *pTsdb);
void tsdbCloseCache(STsdb *pTsdb);
-int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *row);
+int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, SRow **aRow);
+int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData);
int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb);
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 516a47606b..9cbe48a0b0 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -385,7 +385,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
SWalReader* pWalReader = pReader->pWalReader;
- uint64_t st = taosGetTimestampMs();
+ int64_t st = taosGetTimestampMs();
while (1) {
int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData);
while (pReader->nextBlk < numOfBlocks) {
@@ -413,7 +413,8 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
pReader->msg.msgStr = NULL;
- if (taosGetTimestampMs() - st > 1000) {
+ int64_t elapsed = taosGetTimestampMs() - st;
+ if(elapsed > 1000 || elapsed < 0){
return false;
}
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index c0f58fc3ec..5522d06263 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -71,7 +71,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
if (varTbName != NULL && varTbName != (void*)-1) {
name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
memcpy(name, varDataVal(varTbName), varDataLen(varTbName));
- if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name) && groupId != 0 && stbFullName) {
+ if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name, groupId) && groupId != 0 && stbFullName) {
buildCtbNameAddGroupId(stbFullName, name, groupId);
}
} else if (stbFullName) {
@@ -182,7 +182,7 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa
int64_t gid, bool newSubTableRule) {
if (pDataBlock->info.parTbName[0]) {
if (newSubTableRule && !isAutoTableName(pDataBlock->info.parTbName) &&
- !alreadyAddGroupId(pDataBlock->info.parTbName) && gid != 0 && stbFullName) {
+ !alreadyAddGroupId(pDataBlock->info.parTbName, gid) && gid != 0 && stbFullName) {
pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
strcpy(pCreateTableReq->name, pDataBlock->info.parTbName);
buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid);
@@ -713,7 +713,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName);
} else {
if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) &&
- !alreadyAddGroupId(dstTableName) && groupId != 0) {
+ !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) {
tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName);
if(pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER){
buildCtbNameAddGroupId(NULL, dstTableName, groupId);
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index a8daa8f633..f602ee5fe3 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -376,7 +376,7 @@ static SLastCol *tsdbCacheDeserialize(char const *value, size_t size) {
return NULL;
}
- SLastCol* pLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
+ SLastCol *pLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
if (NULL == pLastCol) {
return NULL;
}
@@ -998,46 +998,30 @@ static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal
}
}
-int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow) {
+static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray *updCtxArray) {
+ if (!updCtxArray || TARRAY_SIZE(updCtxArray) == 0) {
+ return 0;
+ }
+
int32_t code = 0;
- // 1, fetch schema
- STSchema *pTSchema = NULL;
- int32_t sver = TSDBROW_SVERSION(pRow);
-
- code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
- if (code != TSDB_CODE_SUCCESS) {
- terrno = code;
- return -1;
- }
-
- // 2, iterate col values into array
- SArray *aColVal = taosArrayInit(32, sizeof(SColVal));
-
- STSDBRowIter iter = {0};
- tsdbRowIterOpen(&iter, pRow, pTSchema);
-
- for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) {
- taosArrayPush(aColVal, pColVal);
- }
-
- tsdbRowClose(&iter);
-
- // 3, build keys & multi get from rocks
- int num_keys = TARRAY_SIZE(aColVal);
+ int num_keys = TARRAY_SIZE(updCtxArray);
SArray *remainCols = NULL;
SLRUCache *pCache = pTsdb->lruCache;
- STsdbRowKey tsdbRowKey = {0};
- tsdbRowGetKey(pRow, &tsdbRowKey);
- SRowKey *pRowKey = &tsdbRowKey.key;
-
taosThreadMutexLock(&pTsdb->lruMutex);
for (int i = 0; i < num_keys; ++i) {
- SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, i);
- int16_t cid = pColVal->cid;
+ SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i);
- SLastKey *key = &(SLastKey){.lflag = LFLAG_LAST_ROW, .uid = uid, .cid = cid};
+ int8_t lflag = updCtx->lflag;
+ SRowKey *pRowKey = &updCtx->tsdbRowKey.key;
+ SColVal *pColVal = &updCtx->colVal;
+
+ if (lflag == LFLAG_LAST && !COL_VAL_IS_VALUE(pColVal)) {
+ continue;
+ }
+
+ SLastKey *key = &(SLastKey){.lflag = lflag, .uid = uid, .cid = pColVal->cid};
size_t klen = ROCKS_KEY_LEN;
LRUHandle *h = taosLRUCacheLookup(pCache, key, klen);
if (h) {
@@ -1053,23 +1037,6 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
}
taosArrayPush(remainCols, &(SIdxKey){i, *key});
}
-
- if (COL_VAL_IS_VALUE(pColVal)) {
- key->lflag = LFLAG_LAST;
- LRUHandle *h = taosLRUCacheLookup(pCache, key, klen);
- if (h) {
- SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h);
- if (tRowKeyCompare(&pLastCol->rowKey, pRowKey) != 1) {
- tsdbCacheUpdateLastCol(pLastCol, pRowKey, pColVal);
- }
- taosLRUCacheRelease(pCache, h, false);
- } else {
- if (!remainCols) {
- remainCols = taosArrayInit(num_keys * 2, sizeof(SIdxKey));
- }
- taosArrayPush(remainCols, &(SIdxKey){i, *key});
- }
- }
}
if (remainCols) {
@@ -1097,99 +1064,60 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
for (int i = 0; i < num_keys; ++i) {
SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i];
- SColVal *pColVal = (SColVal *)TARRAY_DATA(aColVal) + idxKey->idx;
- // SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, idxKey->idx);
+ SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i);
+ SRowKey *pRowKey = &updCtx->tsdbRowKey.key;
+ SColVal *pColVal = &updCtx->colVal;
SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i], values_list_sizes[i]);
SLastCol *PToFree = pLastCol;
- if (IS_LAST_ROW_KEY(idxKey->key)) {
- int32_t cmp_res = 1;
- if (pLastCol) {
- cmp_res = tRowKeyCompare(&pLastCol->rowKey, pRowKey);
- }
+ if (IS_LAST_KEY(idxKey->key) && !COL_VAL_IS_VALUE(pColVal)) {
+ continue;
+ }
- if (NULL == pLastCol || cmp_res < 0 || (cmp_res == 0 && !COL_VAL_IS_NONE(pColVal))) {
- char *value = NULL;
- size_t vlen = 0;
- SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal};
- tsdbCacheSerialize(&lastColTmp, &value, &vlen);
+ int32_t cmp_res = 1;
+ if (pLastCol) {
+ cmp_res = tRowKeyCompare(&pLastCol->rowKey, pRowKey);
+ }
- taosThreadMutexLock(&pTsdb->rCache.rMutex);
+ if (NULL == pLastCol || cmp_res < 0 || (cmp_res == 0 && !COL_VAL_IS_NONE(pColVal))) {
+ char *value = NULL;
+ size_t vlen = 0;
+ SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal};
+ tsdbCacheSerialize(&lastColTmp, &value, &vlen);
- rocksdb_writebatch_put(wb, (char *)&idxKey->key, ROCKS_KEY_LEN, value, vlen);
+ taosThreadMutexLock(&pTsdb->rCache.rMutex);
- taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
+ rocksdb_writebatch_put(wb, (char *)&idxKey->key, ROCKS_KEY_LEN, value, vlen);
- pLastCol = &lastColTmp;
- SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
- *pTmpLastCol = *pLastCol;
- pLastCol = pTmpLastCol;
+ taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
- size_t charge = sizeof(*pLastCol);
- for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) {
- SValue *pValue = &pLastCol->rowKey.pks[i];
- if (IS_VAR_DATA_TYPE(pValue->type)) {
- reallocVarDataVal(pValue);
- charge += pValue->nData;
- }
- }
+ pLastCol = &lastColTmp;
+ SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
+ *pTmpLastCol = *pLastCol;
+ pLastCol = pTmpLastCol;
- if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
- reallocVarData(&pLastCol->colVal);
- charge += pLastCol->colVal.value.nData;
- }
-
- LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge,
- tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState);
- if (status != TAOS_LRU_STATUS_OK) {
- code = -1;
- }
-
- taosMemoryFree(value);
- }
- } else {
- if (COL_VAL_IS_VALUE(pColVal)) {
- if (NULL == pLastCol || (tRowKeyCompare(&pLastCol->rowKey, pRowKey) != 1)) {
- char *value = NULL;
- size_t vlen = 0;
- SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal};
- tsdbCacheSerialize(&lastColTmp, &value, &vlen);
-
- taosThreadMutexLock(&pTsdb->rCache.rMutex);
-
- rocksdb_writebatch_put(wb, (char *)&idxKey->key, ROCKS_KEY_LEN, value, vlen);
-
- taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
-
- pLastCol = &lastColTmp;
- SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
- *pTmpLastCol = *pLastCol;
- pLastCol = pTmpLastCol;
-
- size_t charge = sizeof(*pLastCol);
- for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) {
- SValue *pValue = &pLastCol->rowKey.pks[i];
- if (IS_VAR_DATA_TYPE(pValue->type)) {
- reallocVarDataVal(pValue);
- charge += pValue->nData;
- }
- }
-
- if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
- reallocVarData(&pLastCol->colVal);
- charge += pLastCol->colVal.value.nData;
- }
-
- LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge,
- tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState);
- if (status != TAOS_LRU_STATUS_OK) {
- code = -1;
- }
-
- taosMemoryFree(value);
+ size_t charge = sizeof(*pLastCol);
+ for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) {
+ SValue *pValue = &pLastCol->rowKey.pks[i];
+ if (IS_VAR_DATA_TYPE(pValue->type)) {
+ reallocVarDataVal(pValue);
+ charge += pValue->nData;
}
}
+
+ if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
+ reallocVarData(&pLastCol->colVal);
+ charge += pLastCol->colVal.value.nData;
+ }
+
+ LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge,
+ tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState);
+ if (status != TAOS_LRU_STATUS_OK) {
+ code = -1;
+ }
+
+ taosMemoryFree(value);
}
taosMemoryFreeClear(PToFree);
@@ -1209,11 +1137,152 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
taosThreadMutexUnlock(&pTsdb->lruMutex);
_exit:
- taosArrayDestroy(aColVal);
- taosMemoryFree(pTSchema);
return code;
}
+int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow,
+ SRow **aRow) {
+ int32_t code = 0;
+
+ // 1. prepare last
+ TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version};
+
+ STSchema *pTSchema = NULL;
+ int32_t sver = TSDBROW_SVERSION(&lRow);
+ SArray *ctxArray = NULL;
+ SSHashObj *iColHash = NULL;
+
+ code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
+ if (code != TSDB_CODE_SUCCESS) {
+ terrno = code;
+ goto _exit;
+ }
+
+ TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version};
+ int32_t nCol = pTSchema->numOfCols;
+
+ ctxArray = taosArrayInit(nCol, sizeof(SLastUpdateCtx));
+ iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
+
+ // 1. prepare by lrow
+ STsdbRowKey tsdbRowKey = {0};
+ tsdbRowGetKey(&lRow, &tsdbRowKey);
+
+ STSDBRowIter iter = {0};
+ tsdbRowIterOpen(&iter, &lRow, pTSchema);
+ int32_t iCol = 0;
+ for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) {
+ SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal};
+ taosArrayPush(ctxArray, &updateCtx);
+
+ if (!COL_VAL_IS_VALUE(pColVal)) {
+ tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0);
+ continue;
+ }
+ updateCtx.lflag = LFLAG_LAST;
+ taosArrayPush(ctxArray, &updateCtx);
+ }
+ tsdbRowClose(&iter);
+
+ // 2. prepare by the other rows
+ for (int32_t iRow = nRow - 2; iRow >= 0; --iRow) {
+ if (tSimpleHashGetSize(iColHash) == 0) {
+ break;
+ }
+
+ tRow.pTSRow = aRow[iRow];
+
+ STsdbRowKey tsdbRowKey = {0};
+ tsdbRowGetKey(&tRow, &tsdbRowKey);
+
+ void *pIte = NULL;
+ int32_t iter = 0;
+ while ((pIte = tSimpleHashIterate(iColHash, pIte, &iter)) != NULL) {
+ int32_t iCol = ((int32_t *)pIte)[0];
+ SColVal colVal = COL_VAL_NONE(0, 0);
+ tsdbRowGetColVal(&tRow, pTSchema, iCol, &colVal);
+
+ if (COL_VAL_IS_VALUE(&colVal)) {
+ SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal};
+ taosArrayPush(ctxArray, &updateCtx);
+ tSimpleHashIterateRemove(iColHash, &iCol, sizeof(iCol), &pIte, &iter);
+ }
+ }
+ }
+
+ // 3. do update
+ tsdbCacheUpdate(pTsdb, suid, uid, ctxArray);
+
+_exit:
+ taosMemoryFreeClear(pTSchema);
+ taosArrayDestroy(ctxArray);
+ tSimpleHashCleanup(iColHash);
+ return code;
+}
+
+int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData) {
+ int32_t code = 0;
+
+ TSDBROW lRow = tsdbRowFromBlockData(pBlockData, pBlockData->nRow - 1);
+
+ STSchema *pTSchema = NULL;
+ int32_t sver = TSDBROW_SVERSION(&lRow);
+ SArray *ctxArray = NULL;
+
+ code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
+ if (code != TSDB_CODE_SUCCESS) {
+ terrno = code;
+ goto _exit;
+ }
+
+ ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx));
+
+ // 1. prepare last
+ TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0);
+
+ for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) {
+ SColData *pColData = &pBlockData->aColData[iColData];
+ if ((pColData->flag & HAS_VALUE) != HAS_VALUE) {
+ continue;
+ }
+
+ for (tRow.iRow = pBlockData->nRow - 1; tRow.iRow >= 0; --tRow.iRow) {
+ STsdbRowKey tsdbRowKey = {0};
+ tsdbRowGetKey(&tRow, &tsdbRowKey);
+
+ uint8_t colType = tColDataGetBitValue(pColData, tRow.iRow);
+ if (colType == 2) {
+ SColVal colVal = COL_VAL_NONE(pColData->cid, pColData->type);
+ tColDataGetValue(pColData, tRow.iRow, &colVal);
+
+ SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal};
+ taosArrayPush(ctxArray, &updateCtx);
+ break;
+ }
+ }
+ }
+
+ // 2. prepare last row
+ STsdbRowKey tsdbRowKey = {0};
+ tsdbRowGetKey(&lRow, &tsdbRowKey);
+
+ STSDBRowIter iter = {0};
+ tsdbRowIterOpen(&iter, &lRow, pTSchema);
+ for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) {
+ SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal};
+ taosArrayPush(ctxArray, &updateCtx);
+ }
+ tsdbRowClose(&iter);
+
+ // 3. do update
+ tsdbCacheUpdate(pTsdb, suid, uid, ctxArray);
+
+_exit:
+ taosMemoryFreeClear(pTSchema);
+ taosArrayDestroy(ctxArray);
+ return 0;
+}
+
static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols,
int nCols, int16_t *slotIds);
@@ -1487,7 +1556,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
taosArraySet(pLastArray, idxKey->idx, pLastCol);
// taosArrayRemove(remainCols, i);
- if (!pTmpColArray) {
+ if (/*!pTmpColArray*/ lastTmpIndexArray && !lastTmpColArray) {
+ continue;
+ }
+ if (/*!pTmpColArray*/ lastrowTmpIndexArray && lastrowTmpColArray) {
continue;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
index 40cf9adde1..211855b245 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@@ -629,14 +629,12 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
SMemSkipListNode *pos[SL_MAX_LEVEL];
TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0);
STsdbRowKey key;
- TSDBROW lRow; // last row
// first row
tsdbRowGetKey(&tRow, &key);
tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0))) goto _exit;
pTbData->minKey = TMIN(pTbData->minKey, key.key.ts);
- lRow = tRow;
// remain row
++tRow.iRow;
@@ -653,7 +651,6 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
}
if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1))) goto _exit;
- lRow = tRow;
++tRow.iRow;
}
@@ -664,7 +661,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
}
if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) {
- tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow);
+ tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData);
}
// SMemTable
@@ -688,7 +685,6 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
SMemSkipListNode *pos[SL_MAX_LEVEL];
TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version};
int32_t iRow = 0;
- TSDBROW lRow;
// backward put first data
tRow.pTSRow = aRow[iRow++];
@@ -696,7 +692,6 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0);
if (code) goto _exit;
- lRow = tRow;
pTbData->minKey = TMIN(pTbData->minKey, key.key.ts);
@@ -717,8 +712,6 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1);
if (code) goto _exit;
- lRow = tRow;
-
iRow++;
}
}
@@ -727,7 +720,7 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
pTbData->maxKey = key.key.ts;
}
if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) {
- tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow);
+ tsdbCacheRowFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, version, nRow, aRow);
}
// SMemTable
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index fcd9669f9a..58075cf0ac 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -601,17 +601,21 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
STColumn *pTColumn = &pTSchema->columns[iCol];
SValue value;
- ASSERT(iCol > 0);
-
if (pRow->type == TSDBROW_ROW_FMT) {
tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal);
} else if (pRow->type == TSDBROW_COL_FMT) {
- SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId);
-
- if (pColData) {
- tColDataGetValue(pColData, pRow->iRow, pColVal);
+ if (iCol == 0) {
+ *pColVal =
+ COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID,
+ ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, .val = pRow->pBlockData->aTSKEY[pRow->iRow]}));
} else {
- *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
+ SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId);
+
+ if (pColData) {
+ tColDataGetValue(pColData, pRow->iRow, pColVal);
+ } else {
+ *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
+ }
}
} else {
ASSERT(0);
@@ -1817,4 +1821,4 @@ uint32_t tsdbCvtTimestampAlg(uint32_t alg) {
DEFINE_VAR(alg)
return 0;
-}
\ No newline at end of file
+}
diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c
index 2cba6e3241..29907e6f1f 100644
--- a/source/libs/executor/src/eventwindowoperator.c
+++ b/source/libs/executor/src/eventwindowoperator.c
@@ -202,6 +202,7 @@ static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator) {
}
eventWindowAggImpl(pOperator, pInfo, pBlock);
+ doFilter(pRes, pSup->pFilterInfo, NULL);
if (pRes->info.rows >= pOperator->resultInfo.threshold) {
return pRes;
}
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index e2ea6c1374..2a5e594ba3 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -1424,7 +1424,7 @@ int main(int argc, char *argv[]) {
printf("failed to start since init log error\n");
}
- if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) {
+ if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
fnError("failed to start since read config error");
return -2;
}
diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c
index eef6cdc3d3..aa8b88b738 100644
--- a/source/libs/function/test/runUdf.c
+++ b/source/libs/function/test/runUdf.c
@@ -127,7 +127,7 @@ int aggregateFuncTest() {
int main(int argc, char *argv[]) {
parseArgs(argc, argv);
initLog();
- if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) {
+ if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
fnError("failed to start since read config error");
return -1;
}
diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index 9ae9dfffd7..5399aa7f09 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -167,7 +167,7 @@ static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool demandDb)
}
} else {
trimEscape(pDbName);
- if (pDbName->n >= TSDB_DB_NAME_LEN) {
+ if (pDbName->n >= TSDB_DB_NAME_LEN || pDbName->n == 0) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pDbName->z);
}
}
@@ -176,7 +176,7 @@ static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool demandDb)
static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) {
trimEscape(pTableName);
- if (NULL != pTableName && pTableName->n >= TSDB_TABLE_NAME_LEN) {
+ if (NULL != pTableName && pTableName->type != TK_NK_NIL && (pTableName->n >= TSDB_TABLE_NAME_LEN || pTableName->n == 0)) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z);
return false;
}
@@ -185,7 +185,7 @@ static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) {
static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) {
trimEscape(pColumnName);
- if (NULL != pColumnName && pColumnName->n >= TSDB_COL_NAME_LEN) {
+ if (NULL != pColumnName && pColumnName->type != TK_NK_NIL && (pColumnName->n >= TSDB_COL_NAME_LEN || pColumnName->n == 0)) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z);
return false;
}
@@ -203,7 +203,7 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) {
static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) {
trimEscape(pTopicName);
- if (pTopicName->n >= TSDB_TOPIC_NAME_LEN) {
+ if (pTopicName->n >= TSDB_TOPIC_NAME_LEN || pTopicName->n == 0) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTopicName->z);
return false;
}
@@ -221,7 +221,7 @@ static bool checkCGroupName(SAstCreateContext* pCxt, SToken* pCGroup) {
static bool checkViewName(SAstCreateContext* pCxt, SToken* pViewName) {
trimEscape(pViewName);
- if (pViewName->n >= TSDB_VIEW_NAME_LEN) {
+ if (pViewName->n >= TSDB_VIEW_NAME_LEN || pViewName->n == 0) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pViewName->z);
return false;
}
@@ -230,7 +230,7 @@ static bool checkViewName(SAstCreateContext* pCxt, SToken* pViewName) {
static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) {
trimEscape(pStreamName);
- if (pStreamName->n >= TSDB_STREAM_NAME_LEN) {
+ if (pStreamName->n >= TSDB_STREAM_NAME_LEN || pStreamName->n == 0) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pStreamName->z);
return false;
}
@@ -252,6 +252,8 @@ static bool checkTsmaName(SAstCreateContext* pCxt, SToken* pTsmaToken) {
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
} else if (pTsmaToken->n >= TSDB_TABLE_NAME_LEN - strlen(TSMA_RES_STB_POSTFIX)) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSMA_NAME_TOO_LONG);
+ } else if (pTsmaToken->n == 0) {
+ pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTsmaToken->z);
}
return pCxt->errCode == TSDB_CODE_SUCCESS;
}
@@ -1935,6 +1937,7 @@ SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int
SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, SToken* pTagName, SNode* pVal) {
CHECK_PARSER_STATUS(pCxt);
if (!checkColumnName(pCxt, pTagName)) {
+ nodesDestroyNode(pVal);
return NULL;
}
SAlterTableStmt* pStmt = (SAlterTableStmt*)nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
@@ -1946,6 +1949,7 @@ SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, SToken
}
SNode* setAlterSuperTableType(SNode* pStmt) {
+ if (!pStmt) return NULL;
setNodeType(pStmt, QUERY_NODE_ALTER_SUPER_TABLE_STMT);
return pStmt;
}
@@ -2436,6 +2440,7 @@ SNode* createRestoreComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, c
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery) {
CHECK_PARSER_STATUS(pCxt);
if (!checkTopicName(pCxt, pTopicName)) {
+ nodesDestroyNode(pQuery);
return NULL;
}
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
@@ -2721,6 +2726,8 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken
SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols) {
CHECK_PARSER_STATUS(pCxt);
if (!checkStreamName(pCxt, pStreamName)) {
+ nodesDestroyNode(pQuery);
+ nodesDestroyNode(pOptions);
return NULL;
}
SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT);
diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c
index 370514e1fc..22f274b21c 100644
--- a/source/libs/parser/src/parInsertSql.c
+++ b/source/libs/parser/src/parInsertSql.c
@@ -2393,7 +2393,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif
// no data in the sql string anymore.
if (0 == pTbName->n) {
if (0 != pTbName->type && '\0' != pStmt->pSql[0]) {
- return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", pTbName->z);
+ return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pTbName->z);
}
if (0 == pStmt->totalRowsNum && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) {
diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c
index 5ad9feddbc..7c43113ad2 100644
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@@ -103,6 +103,9 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c
if (pTableName->n >= TSDB_TABLE_NAME_LEN) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
+ if (pTableName->n == 0) {
+ return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "invalid table name");
+ }
char name[TSDB_TABLE_FNAME_LEN] = {0};
strncpy(name, pTableName->z, pTableName->n);
@@ -111,6 +114,8 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c
if (dbName == NULL) {
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
+ if (name[0] == '\0')
+ return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, msg4);
code = tNameSetDbName(pName, acctId, dbName, strlen(dbName));
if (code != TSDB_CODE_SUCCESS) {
@@ -732,6 +737,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
}
}
} else {
+ bool hasTs = false;
for (int i = 0; i < numFields; i++) {
for (int j = 0; j < boundInfo->numOfBound; j++) {
SSchema* pColSchema = &pSchema[j];
@@ -743,6 +749,10 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
goto end;
}
+ if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+ hasTs = true;
+ }
+
int8_t* offset = pStart;
if (IS_VAR_DATA_TYPE(pColSchema->type)) {
pStart += numOfRows * sizeof(int32_t);
@@ -768,6 +778,12 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
}
}
+ if(!hasTs){
+ if (errstr != NULL) snprintf(errstr, errstrLen, "timestamp column(primary key) not found in raw data");
+ ret = TSDB_CODE_INVALID_PARA;
+ goto end;
+ }
+
for (int c = 0; c < boundInfo->numOfBound; ++c) {
if (boundInfo->pColIndex[c] != -1) {
SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c);
diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c
index faa90dcbf8..11e9350e14 100644
--- a/source/libs/qworker/src/qwMsg.c
+++ b/source/libs/qworker/src/qwMsg.c
@@ -370,12 +370,12 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran
if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) {
QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask);
tFreeSSubQueryMsg(&msg);
- QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED);
+ QW_ERR_RET(TSDB_CODE_GRANT_VIEW_EXPIRED);
}
if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) {
QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask);
tFreeSSubQueryMsg(&msg);
- QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED);
+ QW_ERR_RET(TSDB_CODE_GRANT_AUDIT_EXPIRED);
}
}
}
diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h
index 704bc9a2f2..bb6362ff73 100644
--- a/source/libs/stream/inc/streamBackendRocksdb.h
+++ b/source/libs/stream/inc/streamBackendRocksdb.h
@@ -247,6 +247,7 @@ int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId);
int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId);
int32_t taskDbBuildSnap(void* arg, SArray* pSnap);
+int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo);
int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId);
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index db13da5f3b..2e65c9ef16 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -146,6 +146,9 @@ static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter,
static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName,
rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt);
+void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp);
+void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp);
+
#define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX));
int32_t copyFiles(const char* src, const char* dst);
uint32_t nextPow2(uint32_t x);
@@ -1010,6 +1013,50 @@ int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*
}
#endif
+int chkpIdComp(const void* a, const void* b) {
+ int64_t x = *(int64_t*)a;
+ int64_t y = *(int64_t*)b;
+ if (x == y) return 0;
+
+ return x < y ? -1 : 1;
+}
+int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) {
+ int32_t code = 0;
+ char* pChkpDir = taosMemoryCalloc(1, 256);
+
+ sprintf(pChkpDir, "%s%s%s", pBackend->path, TD_DIRSEP, "checkpoints");
+ if (!taosIsDir(pChkpDir)) {
+ taosMemoryFree(pChkpDir);
+ return 0;
+ }
+ TdDirPtr pDir = taosOpenDir(pChkpDir);
+ if (pDir == NULL) {
+ taosMemoryFree(pChkpDir);
+ return 0;
+ }
+ TdDirEntryPtr de = NULL;
+ while ((de = taosReadDir(pDir)) != NULL) {
+ if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue;
+
+ if (taosDirEntryIsDir(de)) {
+ char checkpointPrefix[32] = {0};
+ int64_t checkpointId = 0;
+
+ int ret = sscanf(taosGetDirEntryName(de), "checkpoint%" PRId64 "", &checkpointId);
+ if (ret == 1) {
+ taosArrayPush(pBackend->chkpSaved, &checkpointId);
+ }
+ } else {
+ continue;
+ }
+ }
+ taosArraySort(pBackend->chkpSaved, chkpIdComp);
+
+ taosMemoryFree(pChkpDir);
+ taosCloseDir(&pDir);
+
+ return 0;
+}
int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) {
SArray* pHandle = taosArrayInit(8, POINTER_BYTES);
for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) {
@@ -1043,8 +1090,7 @@ int32_t chkpDoDbCheckpoint(rocksdb_t* db, char* path) {
taosMemoryFreeClear(err);
goto _ERROR;
}
-
- rocksdb_checkpoint_create(cp, path, 64 << 20, &err);
+ rocksdb_checkpoint_create(cp, path, UINT64_MAX, &err);
if (err != NULL) {
stError("failed to do checkpoint at:%s, reason:%s", path, err);
taosMemoryFreeClear(err);
@@ -1056,7 +1102,6 @@ _ERROR:
return code;
}
int32_t chkpPreFlushDb(rocksdb_t* db, rocksdb_column_family_handle_t** cf, int32_t nCf) {
- if (nCf == 0) return 0;
int code = 0;
char* err = NULL;
@@ -1109,9 +1154,12 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
STaskDbWrapper* pTaskDb = *(STaskDbWrapper**)pIter;
taskDbAddRef(pTaskDb);
- code = taskDbDoCheckpoint(pTaskDb, pTaskDb->chkpId);
+ int64_t chkpId = pTaskDb->chkpId;
+ code = taskDbDoCheckpoint(pTaskDb, chkpId);
taskDbRemoveRef(pTaskDb);
+ taskDbRefChkp(pTaskDb, pTaskDb->chkpId);
+
SStreamTask* pTask = pTaskDb->pTask;
SStreamTaskSnap snap = {.streamId = pTask->id.streamId,
.taskId = pTask->id.taskId,
@@ -1124,6 +1172,28 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
return code;
}
+int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) {
+ if (pSnapInfo == NULL) return 0;
+ SStreamMeta* pMeta = arg;
+ int32_t code = 0;
+ taosThreadMutexLock(&pMeta->backendMutex);
+
+ char buf[128] = {0};
+ for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) {
+ SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfo, i);
+ sprintf(buf, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)pSnap->taskId);
+ STaskDbWrapper* pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf));
+ if (pTaskDb == NULL) {
+ stWarn("stream backend:%p failed to find task db, streamId:% " PRId64 "", pMeta, pSnap->streamId);
+ continue;
+ }
+ memset(buf, 0, sizeof(buf));
+
+ taskDbUnRefChkp(pTaskDb, pSnap->chkpId);
+ }
+ taosThreadMutexUnlock(&pMeta->backendMutex);
+ return 0;
+}
#ifdef BUILD_NO_CALL
int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId) {
// if (arg == NULL) return 0;
@@ -1192,20 +1262,23 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) {
int32_t nCf = chkpGetAllDbCfHandle2(pTaskDb, &ppCf);
stDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf);
- if ((code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf)) == 0) {
- if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) {
- stError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir);
- } else {
- stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir,
- taosGetTimestampMs() - st);
- }
+ int64_t written = atomic_load_64(&pTaskDb->dataWritten);
+
+ if (written > 0) {
+ stDebug("stream backend:%p start to flush db at:%s, data written:%" PRId64 "", pTaskDb, pChkpIdDir, written);
+ code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf);
} else {
- stError("stream backend:%p failed to flush db at:%s", pTaskDb, pChkpIdDir);
+ stDebug("stream backend:%p not need flush db at:%s, data written:%" PRId64 "", pTaskDb, pChkpIdDir, written);
+ }
+ if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) {
+ stError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir);
+ } else {
+ stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir,
+ taosGetTimestampMs() - st);
}
code = chkpMayDelObsolete(pTaskDb, chkpId, pChkpDir);
- pTaskDb->dataWritten = 0;
-
+ atomic_store_64(&pTaskDb->dataWritten, 0);
pTaskDb->chkpId = chkpId;
_EXIT:
@@ -1846,7 +1919,7 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) {
rocksdb_options_set_create_if_missing(opts, 1);
rocksdb_options_set_create_missing_column_families(opts, 1);
// rocksdb_options_set_max_total_wal_size(opts, dbMemLimit);
- rocksdb_options_set_recycle_log_file_num(opts, 6);
+ // rocksdb_options_set_ecycle_log_file_num(opts, 6);
rocksdb_options_set_max_write_buffer_number(opts, 3);
rocksdb_options_set_info_log_level(opts, 1);
rocksdb_options_set_db_write_buffer_size(opts, 256 << 20);
@@ -1900,11 +1973,32 @@ void taskDbInitChkpOpt(STaskDbWrapper* pTaskDb) {
pTaskDb->chkpId = -1;
pTaskDb->chkpCap = 4;
pTaskDb->chkpSaved = taosArrayInit(4, sizeof(int64_t));
+ taskDbLoadChkpInfo(pTaskDb);
+
pTaskDb->chkpInUse = taosArrayInit(4, sizeof(int64_t));
taosThreadRwlockInit(&pTaskDb->chkpDirLock, NULL);
}
+void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) {
+ taosThreadRwlockWrlock(&pTaskDb->chkpDirLock);
+ taosArrayPush(pTaskDb->chkpInUse, &chkp);
+ taosArraySort(pTaskDb->chkpInUse, chkpIdComp);
+ taosThreadRwlockUnlock(&pTaskDb->chkpDirLock);
+}
+
+void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) {
+ taosThreadRwlockWrlock(&pTaskDb->chkpDirLock);
+ for (int i = 0; i < taosArrayGetSize(pTaskDb->chkpInUse); i++) {
+ int64_t* p = taosArrayGet(pTaskDb->chkpInUse, i);
+ if (*p == chkp) {
+ taosArrayRemove(pTaskDb->chkpInUse, i);
+ break;
+ }
+ }
+ taosThreadRwlockUnlock(&pTaskDb->chkpDirLock);
+}
+
void taskDbDestroyChkpOpt(STaskDbWrapper* pTaskDb) {
taosArrayDestroy(pTaskDb->chkpSaved);
taosArrayDestroy(pTaskDb->chkpInUse);
@@ -2131,15 +2225,18 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64
return code;
}
int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list) {
+ int32_t code = -1;
STaskDbWrapper* pDb = arg;
ECHECKPOINT_BACKUP_TYPE utype = type;
+ taskDbRefChkp(pDb, chkpId);
if (utype == DATA_UPLOAD_RSYNC) {
- return taskDbGenChkpUploadData__rsync(pDb, chkpId, path);
+ code = taskDbGenChkpUploadData__rsync(pDb, chkpId, path);
} else if (utype == DATA_UPLOAD_S3) {
- return taskDbGenChkpUploadData__s3(pDb, mgt, chkpId, path, list);
+ code = taskDbGenChkpUploadData__s3(pDb, mgt, chkpId, path, list);
}
- return -1;
+ taskDbUnRefChkp(pDb, chkpId);
+ return code;
}
int32_t taskDbOpenCfByKey(STaskDbWrapper* pDb, const char* key) {
@@ -2563,7 +2660,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe
break; \
} \
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \
- wrapper->dataWritten += 1; \
+ atomic_add_fetch_64(&wrapper->dataWritten, 1); \
char toString[128] = {0}; \
if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \
int32_t klen = ginitDict[i].enFunc((void*)key, buf); \
@@ -2640,7 +2737,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe
break; \
} \
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \
- wrapper->dataWritten += 1; \
+ atomic_add_fetch_64(&wrapper->dataWritten, 1); \
char toString[128] = {0}; \
if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \
int32_t klen = ginitDict[i].enFunc((void*)key, buf); \
@@ -2681,7 +2778,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) {
stDebug("streamStateClear_rocksdb");
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
- wrapper->dataWritten += 1;
+ atomic_add_fetch_64(&wrapper->dataWritten, 1);
char sKeyStr[128] = {0};
char eKeyStr[128] = {0};
@@ -3707,7 +3804,7 @@ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rock
int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb_writebatch_t* pBatch, void* key,
void* val, int32_t vlen, int64_t ttl) {
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
- wrapper->dataWritten += 1;
+ atomic_add_fetch_64(&wrapper->dataWritten, 1);
int i = streamStateGetCfIdx(pState, cfKeyName);
if (i < 0) {
@@ -3741,7 +3838,8 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb
int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(val, vlen, ttl, &ttlV);
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
- wrapper->dataWritten += 1;
+
+ atomic_add_fetch_64(&wrapper->dataWritten, 1);
rocksdb_column_family_handle_t* pCf = wrapper->pCf[ginitDict[cfIdx].idx];
rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen);
@@ -3760,7 +3858,7 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb
int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) {
char* err = NULL;
STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
- wrapper->dataWritten += 1;
+ atomic_add_fetch_64(&wrapper->dataWritten, 1);
rocksdb_write(wrapper->db, wrapper->writeOpt, (rocksdb_writebatch_t*)pBatch, &err);
if (err != NULL) {
stError("streamState failed to write batch, err:%s", err);
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 58c6e19581..cc9105ffdc 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -494,7 +494,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
if (pDataBlock->info.parTbName[0]) {
if(pTask->subtableWithoutMd5 != 1 &&
!isAutoTableName(pDataBlock->info.parTbName) &&
- !alreadyAddGroupId(pDataBlock->info.parTbName) &&
+ !alreadyAddGroupId(pDataBlock->info.parTbName, groupId) &&
groupId != 0){
if(pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER){
buildCtbNameAddGroupId(NULL, pDataBlock->info.parTbName, groupId);
diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index 1800324cc8..eb67e61c4c 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -74,7 +74,9 @@ struct SStreamSnapHandle {
int32_t currFileIdx;
char* metaPath;
+ void* pMeta;
SArray* pDbSnapSet;
+ SArray* pSnapInfoSet;
int32_t currIdx;
int8_t delFlag; // 0 : not del, 1: del
};
@@ -140,7 +142,9 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) {
return taosOpenFile(fullname, opt);
}
-int32_t streamTaskDbGetSnapInfo(void* arg, char* path, SArray* pSnap) { return taskDbBuildSnap(arg, pSnap); }
+int32_t streamCreateTaskDbSnapInfo(void* arg, char* path, SArray* pSnap) { return taskDbBuildSnap(arg, pSnap); }
+
+int32_t streamDestroyTasdDbSnapInfo(void* arg, SArray* snap) { return taskDbDestroySnap(arg, snap); }
void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) {
if (qDebugFlag & DEBUG_DEBUG) {
@@ -291,30 +295,26 @@ void snapFileDestroy(SBackendSnapFile2* pSnap) {
}
int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta) {
// impl later
- SArray* pSnapSet = taosArrayInit(4, sizeof(SStreamTaskSnap));
- int32_t code = streamTaskDbGetSnapInfo(pMeta, path, pSnapSet);
+ SArray* pSnapInfoSet = taosArrayInit(4, sizeof(SStreamTaskSnap));
+ int32_t code = streamCreateTaskDbSnapInfo(pMeta, path, pSnapInfoSet);
if (code != 0) {
return -1;
}
SArray* pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2));
- for (int32_t i = 0; i < taosArrayGetSize(pSnapSet); i++) {
- SStreamTaskSnap* pSnap = taosArrayGet(pSnapSet, i);
+ for (int32_t i = 0; i < taosArrayGetSize(pSnapInfoSet); i++) {
+ SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfoSet, i);
SBackendSnapFile2 snapFile = {0};
code = streamBackendSnapInitFile(path, pSnap, &snapFile);
ASSERT(code == 0);
taosArrayPush(pDbSnapSet, &snapFile);
}
- for (int32_t i = 0; i < taosArrayGetSize(pSnapSet); i++) {
- SStreamTaskSnap* pSnap = taosArrayGet(pSnapSet, i);
- taosMemoryFree(pSnap->dbPrefixPath);
- }
- taosArrayDestroy(pSnapSet);
-
pHandle->pDbSnapSet = pDbSnapSet;
+ pHandle->pSnapInfoSet = pSnapInfoSet;
pHandle->currIdx = 0;
+ pHandle->pMeta = pMeta;
return 0;
_err:
@@ -333,6 +333,14 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
}
taosArrayDestroy(handle->pDbSnapSet);
}
+ streamDestroyTasdDbSnapInfo(handle->pMeta, handle->pSnapInfoSet);
+ if (handle->pSnapInfoSet) {
+ for (int32_t i = 0; i < taosArrayGetSize(handle->pSnapInfoSet); i++) {
+ SStreamTaskSnap* pSnap = taosArrayGet(handle->pSnapInfoSet, i);
+ taosMemoryFree(pSnap->dbPrefixPath);
+ }
+ taosArrayDestroy(handle->pSnapInfoSet);
+ }
taosMemoryFree(handle->metaPath);
return;
}
diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c
index 5a3dc867e6..4635ec340d 100644
--- a/source/util/src/tcompression.c
+++ b/source/util/src/tcompression.c
@@ -2741,7 +2741,7 @@ int32_t tsDecompressTimestamp2(void *pIn, int32_t nIn, int32_t nEle, void *pOut,
int32_t tsCompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf) {
DEFINE_VAR(cmprAlg)
- if (lvl != 0 && lossyFloat) {
+ if (l2 == L2_TSZ && lvl != 0 && lossyFloat) {
return tsCompressFloatLossyImp(pIn, nEle, pOut);
}
FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_FLOAT, 1);
@@ -2760,7 +2760,7 @@ int32_t tsDecompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int
int32_t tsCompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf) {
DEFINE_VAR(cmprAlg)
- if (lvl != 0 && lossyDouble) {
+ if (l2 == L2_TSZ && lvl != 0 && lossyDouble) {
// lossy mode
return tsCompressDoubleLossyImp(pIn, nEle, pOut);
}
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index ad811cc891..0f594af0e9 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -259,7 +259,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY, "Inconsistent encryption key")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ENCRYPT_KEY, "The cluster has not been set properly for database encryption")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED, "The database encryption function grant expired")
// mnode-node
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists")
@@ -499,9 +498,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LACK_OF_BASIC, "Lack of basic functio
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJ_NOT_EXIST, "Grant object not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LAST_ACTIVE_NOT_FOUND, "The historial active code does not match")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MACHINES_MISMATCH, "Cluster machines mismatch with active code")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expire time of optional grant item is too large")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expiration time of optional grant item is too large")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUPLICATED_ACTIVE, "The active code can't be activated repeatedly")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of view has reached the licensed upper limit")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of views has reached the licensed upper limit")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_BASIC_EXPIRED, "License expired for basic functions")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_EXPIRED, "License expired for stream function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED, "License expired for subscription function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_EXPIRED, "License expired for view function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_AUDIT_EXPIRED, "License expired for audit function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CSV_EXPIRED, "License expired for CSV function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED, "License expired for multi-tier storage function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED, "License expired for object storage function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED,"License expired for dual-replica HA function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED, "License expired for database encryption function")
+
// sync
TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout")
diff --git a/tests/army/community/query/show.py b/tests/army/community/query/show.py
index 980e60387a..72b84954cd 100644
--- a/tests/army/community/query/show.py
+++ b/tests/army/community/query/show.py
@@ -118,6 +118,22 @@ class TDTestCase(TBase):
tdSql.query(sql)
tdSql.checkRows(5)
+ # verification for TD-30030
+ tdSql.execute("create table t100 (ts timestamp, pk varchar(20) primary key, c1 varchar(100)) tags (id int)")
+ tdSql.execute("insert into ct1 using t100 tags(1) values('2024-05-17 14:58:52.902', 'a1', '100')")
+ tdSql.execute("insert into ct1 using t100 tags(1) values('2024-05-17 14:58:52.902', 'a2', '200')")
+ tdSql.execute("insert into ct1 using t100 tags(1) values('2024-05-17 14:58:52.902', 'a3', '300')")
+ tdSql.execute("insert into ct2 using t100 tags(2) values('2024-05-17 14:58:52.902', 'a2', '200')")
+ tdSql.execute("create view v100 as select * from t100")
+ tdSql.execute("create view v200 as select * from ct1")
+
+ tdSql.error("show tags from v100", expectErrInfo="Tags can only applied to super table and child table")
+ tdSql.error("show tags from v200", expectErrInfo="Tags can only applied to super table and child table")
+
+ tdSql.execute("create table t200 (ts timestamp, pk varchar(20) primary key, c1 varchar(100))")
+
+ tdSql.error("show tags from t200", expectErrInfo="Tags can only applied to super table and child table")
+
def checkShow(self):
# not support
sql = "show accounts;"
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 7348b1fea0..342389562a 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -323,6 +323,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_insert.py
diff --git a/tests/system-test/0-others/all_insertmode_alltypes.json b/tests/system-test/0-others/all_insertmode_alltypes.json
new file mode 100644
index 0000000000..dd5e373214
--- /dev/null
+++ b/tests/system-test/0-others/all_insertmode_alltypes.json
@@ -0,0 +1,248 @@
+{
+ "filetype": "insert",
+ "cfgdir": "/etc/taos",
+ "host": "127.0.0.1",
+ "port": 6030,
+ "user": "root",
+ "password": "taosdata",
+ "thread_count": 4,
+ "connection_pool_size": 10,
+ "result_file": "./insert_res.txt",
+ "confirm_parameter_prompt": "no",
+ "prepared_rand": 10,
+ "chinese": "no",
+ "insert_interval": 0,
+ "num_of_records_per_req": 10,
+ "databases": [{
+ "dbinfo": {
+ "name": "db_all_insert_mode",
+ "drop": "yes"
+ },
+ "super_tables": [{
+ "name": "sml_json",
+ "child_table_exists":"no",
+ "childtable_count": 8,
+ "childtable_prefix": "stb1_",
+ "escape_character": "no",
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "sml",
+ "line_protocol": "json",
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "insert_rows": 2,
+ "insert_interval": 0,
+ "interlace_rows": 0,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 1,
+ "start_timestamp": "now",
+ "sample_file": "",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [{"type": "BOOL"}],
+ "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}]
+ },{
+ "name": "sml_line",
+ "child_table_exists":"no",
+ "childtable_count": 8,
+ "childtable_prefix": "stb2_",
+ "escape_character": "no",
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "sml",
+ "line_protocol": "line",
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "insert_rows": 2,
+ "insert_interval": 0,
+ "interlace_rows": 0,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 1,
+ "start_timestamp": "now",
+ "sample_file": "",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [{"type": "TINYINT"}],
+ "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}]
+ },{
+ "name": "sml_telnet",
+ "child_table_exists":"no",
+ "childtable_count": 8,
+ "childtable_prefix": "stb3_",
+ "escape_character": "no",
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "sml",
+ "line_protocol": "telnet",
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "insert_rows": 2,
+ "insert_interval": 0,
+ "interlace_rows": 0,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 1,
+ "start_timestamp": "now",
+ "sample_file": "",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [{"type": "SMALLINT"}],
+ "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}]
+ },{
+ "name": "rest",
+ "child_table_exists":"no",
+ "childtable_count": 8,
+ "childtable_prefix": "stb4_",
+ "escape_character": "no",
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "rest",
+ "line_protocol": "json",
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "insert_rows": 2,
+ "insert_interval": 0,
+ "interlace_rows": 0,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 1,
+ "start_timestamp": "now",
+ "sample_file": "",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [{"type": "INT"}],
+ "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}]
+ },{
+ "name": "stmt",
+ "child_table_exists":"no",
+ "childtable_count": 8,
+ "childtable_prefix": "stb5_",
+ "escape_character": "no",
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "stmt",
+ "line_protocol": "json",
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "insert_rows": 2,
+ "insert_interval": 0,
+ "interlace_rows": 0,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 1,
+ "start_timestamp": "now",
+ "sample_file": "",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [{"type": "BIGINT"}],
+ "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}]
+ },{
+ "name": "sml_rest_json",
+ "child_table_exists":"no",
+ "childtable_count": 8,
+ "childtable_prefix": "stb6_",
+ "escape_character": "no",
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "sml-rest",
+ "line_protocol": "json",
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "insert_rows": 2,
+ "insert_interval": 0,
+ "interlace_rows": 0,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 1,
+ "start_timestamp": "now",
+ "sample_file": "",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [{"type": "FLOAT"}],
+ "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}]
+ },{
+ "name": "sml_rest_line",
+ "child_table_exists":"no",
+ "childtable_count": 8,
+ "childtable_prefix": "stb7_",
+ "escape_character": "no",
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "sml-rest",
+ "line_protocol": "line",
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "insert_rows": 2,
+ "insert_interval": 0,
+ "interlace_rows": 0,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 1,
+ "start_timestamp": "now",
+ "sample_file": "",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [{"type": "DOUBLE"}],
+ "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}]
+ },{
+ "name": "sml_rest_telnet",
+ "child_table_exists":"no",
+ "childtable_count": 8,
+ "childtable_prefix": "stb8_",
+ "escape_character": "no",
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "sml-rest",
+ "line_protocol": "telnet",
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "insert_rows": 2,
+ "insert_interval": 0,
+ "interlace_rows": 0,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 1,
+ "start_timestamp": "now",
+ "sample_file": "",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [{"type": "BINARY", "len": 8}],
+ "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}]
+ },{
+ "name": "taosc",
+ "child_table_exists":"no",
+ "childtable_count": 8,
+ "childtable_prefix": "stb9_",
+ "escape_character": "no",
+ "auto_create_table": "no",
+ "batch_create_tbl_num": 10,
+ "data_source": "rand",
+ "insert_mode": "taosc",
+ "line_protocol": "json",
+ "childtable_limit": 0,
+ "childtable_offset": 0,
+ "insert_rows": 2,
+ "insert_interval": 0,
+ "interlace_rows": 0,
+ "disorder_ratio": 0,
+ "disorder_range": 1000,
+ "timestamp_step": 1,
+ "start_timestamp": "now",
+ "sample_file": "",
+ "use_sample_ts": "no",
+ "tags_file": "",
+ "columns": [{"type": "NCHAR", "len": 8}],
+ "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}]
+ }]
+ }]
+}
diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py
index 71adf6eaac..9ba3bd0d2f 100644
--- a/tests/system-test/0-others/compatibility.py
+++ b/tests/system-test/0-others/compatibility.py
@@ -43,6 +43,7 @@ class TDTestCase:
delete from deldata.ct1;
insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a );
flush database deldata;'''
+
def checkProcessPid(self,processName):
i=0
while i<60:
@@ -110,7 +111,8 @@ class TDTestCase:
else:
print(f"{packageName} has been exists")
os.system(f" cd {packagePath} && tar xvf {packageName} && cd {packageTPath} && ./install.sh -e no " )
- tdDnodes.stop(1)
+
+ os.system(f"pkill -9 taosd" )
print(f"start taosd: rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & ")
os.system(f"rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & " )
os.system(f"killall taosadapter" )
@@ -165,32 +167,31 @@ class TDTestCase:
cPath = self.getCfgPath()
dbname = "test"
stb = f"{dbname}.meters"
+ # package_type = "enterprise"
package_type = "community"
self.installTaosd(bPath,cPath,package_type)
# os.system(f"echo 'debugFlag 143' >> {cPath}/taos.cfg ")
tableNumbers=100
- recordNumbers1=100
+ recordNumbers1=1000
recordNumbers2=1000
- # tdsqlF=tdCom.newTdSql()
- # print(tdsqlF)
- # tdsqlF.query(f"SELECT SERVER_VERSION();")
- # print(tdsqlF.query(f"SELECT SERVER_VERSION();"))
- # oldServerVersion=tdsqlF.queryResult[0][0]
- # tdLog.info(f"Base server version is {oldServerVersion}")
- # tdsqlF.query(f"SELECT CLIENT_VERSION();")
- # # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison
- # oldClientVersion=tdsqlF.queryResult[0][0]
- # tdLog.info(f"Base client version is {oldClientVersion}")
- # baseVersion = "3.0.1.8"
-
tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{BASEVERSION}")
- tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -y ")
- os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -y ")
+ tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -O 5 -y ")
+ os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -O 5 -y ")
+ os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'alter database test keep 365000 '")
+ os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'alter database test cachemodel \"both\" '")
+ os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select last(*) from test.meters '")
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database test '")
+ os.system("LD_LIBRARY_PATH=/usr/lib taos -s \"insert into test.d1 values (now+1s, 11, 190, 0.21), (now+2s, 11, 190, 0.21), (now+3s, 11, 190, 0.21), ('2015-07-14 08:39:59.001', 11, 190, 0.21), ('2032-08-14 08:39:59.001 ', 11, 190, 0.21) test.d3 values (now+6s, 11, 190, 0.21), (now+7s, 11, 190, 0.21), (now+8s, 11, 190, 0.21), ('2033-07-14 08:39:59.000', 119, 191, 0.25) test.d3 (ts) values ('2033-07-14 08:39:58.000');\"")
+ os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select last(*) from test.meters '")
+ os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database test '")
+
+ os.system("LD_LIBRARY_PATH=/usr/lib taos -s \"insert into test.d1 values (now+11s, 11, 190, 0.21), (now+12s, 11, 190, 0.21), (now+13s, 11, 190, 0.21), (now+14s, 11, 190, 0.21), (now+15s, 11, 190, 0.21) test.d3 values (now+16s, 11, 190, 0.21), (now+17s, 11, 190, 0.21), (now+18s, 11, 190, 0.21), (now+19s, 119, 191, 0.25) test.d3 (ts) values (now+20s);\"")
os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/com_alltypedata.json -y")
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database curdb '")
+ os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'alter database curdb cachemodel \"both\" '")
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select count(*) from curdb.meters '")
+ os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select last(*) from curdb.meters '")
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select sum(fc) from curdb.meters '")
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select avg(ic) from curdb.meters '")
os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select min(ui) from curdb.meters '")
@@ -211,7 +212,8 @@ class TDTestCase:
os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists {stable_topic} as stable test.meters where tbname like \\"d3\\";" ')
select_topic = "select_test_meters_topic"
- os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists {select_topic} as select current,voltage,phase from test.meters where voltage >= 170;" ')
+ topic_select_sql = "select current,voltage,phase from test.meters where voltage >= 10;"
+ os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists {select_topic} as {topic_select_sql}" ')
os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ')
os.system(f" /usr/bin/taosadapter --version " )
@@ -243,37 +245,12 @@ class TDTestCase:
break
consumer.close()
- # consumer_dict2 = {
- # "group.id": "g2",
- # "td.connect.websocket.scheme": "ws",
- # "td.connect.user": "root",
- # "td.connect.pass": "taosdata",
- # "auto.offset.reset": "earliest",
- # "enable.auto.commit": "false",
- # }
- # consumer = taosws.Consumer(consumer_dict2)
- # try:
- # consumer.subscribe([db_topic,stable_topic])
- # except TmqError:
- # tdLog.exit(f"subscribe error")
- # first_consumer_rows = 0
- # while True:
- # message = consumer.poll(timeout=1.0)
- # if message:
- # for block in message:
- # first_consumer_rows += block.nrows()
- # else:
- # tdLog.notice("message is null and break")
- # break
- # consumer.commit(message)
- # tdLog.debug(f"topic:{select_topic} ,first consumer rows is {first_consumer_rows} in old version")
- # break
-
-
-
+
tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ")
os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y")
- os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '")
+ os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/all_insertmode_alltypes.json -y")
+
+ # os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '")
os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql")
# add deleted data
@@ -285,9 +262,10 @@ class TDTestCase:
if os.system(cmd) == 0:
raise Exception("failed to execute system command. cmd: %s" % cmd)
- os.system("pkill taosd") # make sure all the data are saved in disk.
+ os.system("pkill -9 taosd") # make sure all the data are saved in disk.
+ os.system("pkill -9 taos")
self.checkProcessPid("taosd")
- os.system("pkill taosadapter") # make sure all the data are saved in disk.
+ os.system("pkill -9 taosadapter") # make sure all the data are saved in disk.
self.checkProcessPid("taosadapter")
tdLog.printNoPrefix("==========step2:update new version ")
@@ -309,13 +287,33 @@ class TDTestCase:
tdLog.info(f"New client version is {nowClientVersion}")
tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}")
+ tdsql.query(f"select last(*) from curdb.meters")
+ tdLog.info(tdsql.queryResult)
+ tdsql.query(f"select * from db_all_insert_mode.sml_json")
+ tdsql.checkRows(16)
+
+ tdsql.query(f"select * from db_all_insert_mode.sml_line")
+ tdsql.checkRows(16)
+ tdsql.query(f"select * from db_all_insert_mode.sml_telnet")
+ tdsql.checkRows(16)
+ tdsql.query(f"select * from db_all_insert_mode.rest")
+ tdsql.checkRows(16)
+ tdsql.query(f"select * from db_all_insert_mode.stmt")
+ tdsql.checkRows(16)
+ tdsql.query(f"select * from db_all_insert_mode.sml_rest_json")
+ tdsql.checkRows(16)
+ tdsql.query(f"select * from db_all_insert_mode.sml_rest_line")
+ tdsql.checkRows(16)
+ tdsql.query(f"select * from db_all_insert_mode.sml_rest_telnet")
+ tdsql.checkRows(16)
+
tdsql.query(f"select count(*) from {stb}")
- tdsql.checkData(0,0,tableNumbers*recordNumbers1)
+ tdsql.checkData(0,0,tableNumbers*recordNumbers1+20)
tdsql.query("show streams;")
tdsql.checkRows(2)
-
+
# checkout db4096
tdsql.query("select count(*) from db4096.stb0")
tdsql.checkData(0,0,50000)
@@ -381,9 +379,78 @@ class TDTestCase:
tdLog.exit("The unordered list is not the same as the ordered list.")
+ # check database test and last
+ # first check
+
+ tdsql.query(f"select last(*) from test.meters group by tbname")
+ tdLog.info(tdsql.queryResult)
+ # tdsql.checkRows(tableNumbers)
+
+ tdsql.query(f"select last_row(*) from test.meters group by tbname")
+ tdLog.info(tdsql.queryResult)
+ # tdsql.checkRows(tableNumbers)
+
+ tdsql.query(f"select last_row(*) from test.meters partition by tbname")
+ tdLog.info(tdsql.queryResult)
+ # tdsql.checkRows(tableNumbers)
+
+ tdsql.query(f"select last(*) from test.meters")
+ tdLog.info(tdsql.queryResult)
+ tdsql.checkData(0,0,"2033-07-14 08:39:59.000")
+ tdsql.checkData(0,1,119)
+ tdsql.checkData(0,2,191)
+ tdsql.checkData(0,3,0.25)
+
+ tdsql.query(f"select last_row(*) from test.meters")
+ tdLog.info(tdsql.queryResult)
+ tdsql.checkData(0,0,"2033-07-14 08:39:59.000")
+ tdsql.checkData(0,1,119)
+ tdsql.checkData(0,2,191)
+ tdsql.checkData(0,3,0.25)
+
+ tdsql.query(f"select last(*) from test.d1")
+ tdLog.info(tdsql.queryResult)
+ tdsql.checkData(0,0,"2032-08-14 08:39:59.001")
+ tdsql.checkData(0,1,11)
+ tdsql.checkData(0,2,190)
+ tdsql.checkData(0,3,0.21)
+
+ # update data and check
+ tdsql.execute("insert into test.d2 values ('2033-07-14 08:39:59.002', 139, 182, 1.10) (now+2s, 12, 191, 0.22) test.d2 (ts) values ('2033-07-14 08:39:59.003');")
+ tdsql.execute("insert into test.d2 values (now+5s, 4.3, 104, 0.4);")
+
+ tdsql.query(f"select last(*) from test.meters")
+ tdLog.info(tdsql.queryResult)
+ tdsql.checkData(0,0,"2033-07-14 08:39:59.003")
+ tdsql.checkData(0,1,139)
+ tdsql.checkData(0,2,182)
+ tdsql.checkData(0,3,1.10)
+
+ # repeately insert data and check
+ tdsql.execute("insert into test.d1 values (now+1s, 11, 190, 0.21) (now+2s, 12, 191, 0.22) ('2033-07-14 08:40:01.001', 16, 180, 0.53);")
+
+ tdsql.query(f"select last(*) from test.d1")
+ tdLog.info(tdsql.queryResult)
+ tdsql.checkData(0,0,"2033-07-14 08:40:01.001")
+ tdsql.checkData(0,1,16)
+ tdsql.checkData(0,2,180)
+ tdsql.checkData(0,3,0.53)
+
+ tdsql.query(f"select last(*) from test.meters")
+ tdLog.info(tdsql.queryResult)
+ tdsql.checkData(0,0,"2033-07-14 08:40:01.001")
+ tdsql.checkData(0,1,16)
+ tdsql.checkData(0,2,180)
+ tdsql.checkData(0,3,0.53)
+
+ tdsql.query(f"select last_row(*) from test.meters")
+ tdLog.info(tdsql.queryResult)
+ tdsql.checkData(0,0,"2033-07-14 08:40:01.001")
+ tdsql.checkData(0,1,16)
+ tdsql.checkData(0,2,180)
+ tdsql.checkData(0,3,0.53)
+
# check tmq
- tdsql.execute("insert into test.d80 values (now+1s, 11, 190, 0.21);")
- tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);")
conn = taos.connect()
consumer = Consumer(
@@ -408,7 +475,7 @@ class TDTestCase:
print("consumer has completed and break")
break
consumer.close()
- tdsql.query("select current,voltage,phase from test.meters where voltage >= 170;")
+ tdsql.query(f"{topic_select_sql}")
all_rows = tdsql.queryRows
if consumer_rows < all_rows - first_consumer_rows :
tdLog.exit(f"consumer rows is {consumer_rows}, less than {all_rows - first_consumer_rows}")
diff --git a/tests/system-test/0-others/empty_identifier.py b/tests/system-test/0-others/empty_identifier.py
new file mode 100644
index 0000000000..a576781d86
--- /dev/null
+++ b/tests/system-test/0-others/empty_identifier.py
@@ -0,0 +1,180 @@
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+from datetime import datetime
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+# from tmqCommon import *
+
+COMPARE_DATA = 0
+COMPARE_LEN = 1
+
+class TDTestCase:
+ def __init__(self):
+ self.vgroups = 4
+ self.ctbNum = 10
+ self.rowsPerTbl = 10000
+ self.duraion = '1h'
+
+ def init(self, conn, logSql, replicaVar=1):
+ self.replicaVar = int(replicaVar)
+ tdLog.debug(f"start to excute {__file__}")
+ tdSql.init(conn.cursor(), True)
+
+ def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'):
+ if dropFlag == 1:
+ tsql.execute("drop database if exists %s"%(dbName))
+
+ tsql.execute("create database if not exists %s vgroups %d replica %d duration %s"%(dbName, vgroups, replica, duration))
+ tdLog.debug("complete to create database %s"%(dbName))
+ return
+
+ def create_stable(self,tsql, paraDict):
+ colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"])
+ tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"])
+ sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString)
+ tdLog.debug("%s"%(sqlString))
+ tsql.execute(sqlString)
+ return
+
+ def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0):
+ for i in range(ctbNum):
+ sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \
+ (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx)
+ tsql.execute(sqlString)
+
+ tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
+ return
+
+ def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep):
+ tdLog.debug("start to insert data ............")
+ tsql.execute("use %s" %dbName)
+ pre_insert = "insert into "
+ sql = pre_insert
+
+ for i in range(ctbNum):
+ rowsBatched = 0
+ sql += " %s%d values "%(ctbPrefix,i)
+ for j in range(rowsPerTbl):
+ if (i < ctbNum/2):
+ sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10)
+ else:
+ sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10)
+ rowsBatched += 1
+ if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
+ tsql.execute(sql)
+ rowsBatched = 0
+ if j < rowsPerTbl - 1:
+ sql = "insert into %s%d values " %(ctbPrefix,i)
+ else:
+ sql = "insert into "
+ if sql != pre_insert:
+ tsql.execute(sql)
+ tdLog.debug("insert data ............ [OK]")
+ return
+
+ def prepareTestEnv(self):
+ tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+ paraDict = {'dbName': 'test',
+ 'dropFlag': 1,
+ 'vgroups': 2,
+ 'stbName': 'meters',
+ 'colPrefix': 'c',
+ 'tagPrefix': 't',
+ 'colSchema': [{'type': 'INT', 'count':1},
+ {'type': 'BIGINT', 'count':1},
+ {'type': 'FLOAT', 'count':1},
+ {'type': 'DOUBLE', 'count':1},
+ {'type': 'smallint', 'count':1},
+ {'type': 'tinyint', 'count':1},
+ {'type': 'bool', 'count':1},
+ {'type': 'binary', 'len':10, 'count':1},
+ {'type': 'nchar', 'len':10, 'count':1}],
+ 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}],
+ 'ctbPrefix': 't',
+ 'ctbStartIdx': 0,
+ 'ctbNum': 100,
+ 'rowsPerTbl': 10000,
+ 'batchNum': 3000,
+ 'startTs': 1537146000000,
+ 'tsStep': 600000}
+
+ paraDict['vgroups'] = self.vgroups
+ paraDict['ctbNum'] = self.ctbNum
+ paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+ tdLog.info("create database")
+ self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion)
+
+ tdLog.info("create stb")
+ self.create_stable(tsql=tdSql, paraDict=paraDict)
+
+ tdLog.info("create child tables")
+ self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \
+ stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\
+ ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"])
+ self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\
+ ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\
+ rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\
+ startTs=paraDict["startTs"],tsStep=paraDict["tsStep"])
+ return
+
+ def run(self):
+ self.prepareTestEnv()
+ self.test_empty_identifier()
+
+ def execute_sql_and_expect_err(self, sql: str, err: int):
+ tdSql.error(sql, err)
+
+ def test_empty_identifier(self):
+ ## invalid identifier
+ sqls = [
+ 'show create table ``',
+ 'show create table test.``',
+ 'create table `` (ts timestamp, c1 int)',
+ 'drop table ``',
+ 'alter table `` add column c2 int',
+ 'select * from ``',
+ 'alter table meters add column `` int',
+ 'alter table meters drop column ``',
+ 'alter table t0 set tag `` = ""',
+ 'alter stable meters add tag `` int',
+ 'alter stable meters rename tag cc ``',
+ 'alter stable meters drop tag ``',
+ 'insert into `` select * from t0',
+ 'insert into t100 using `` tags('', '') values(1,1,1)',
+ 'create view `` as select count(*) from meters interval(10s)',
+ 'create view ``.view1 as select count(*) from meters'
+ 'create tsma `` on meters function(count(c1)) interval(1m)',
+ 'create tsma tsma1 on `` function(count(c1)) interval(1m)',
+ 'create stream `` into st1 as select count(*) from meters interval(10s)',
+ 'create stream stream1 into `` as select count(*) from meters interval(10s)',
+ 'create stream stream1 into st1 as select count(*) from `` interval(10s)',
+ 'create stream stream1 trigger max_delay 100s into st1 as select count(*) from `` interval(10s)',
+ 'drop view ``',
+ 'drop tsma ``',
+ 'drop view ``.st1',
+ 'create topic `` as select count(*) from meters interval(10s)',
+ 'drop topic ``',
+ 'insert into `` values(1,1,1)',
+ ]
+
+ for sql in sqls:
+ self.execute_sql_and_expect_err(sql, -2147473897)
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py
index 4b236c1bce..c63b8af9df 100644
--- a/tests/system-test/2-query/group_partition.py
+++ b/tests/system-test/2-query/group_partition.py
@@ -237,6 +237,9 @@ class TDTestCase:
tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and t2=0;")
tdSql.checkRows(1)
+ tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and t2=0 having count(*) > 10;")
+ tdSql.checkRows(0)
+
tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _rowts>0;")
tdSql.checkRows(nonempty_tb_num)
diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c
index 645cf0e6fe..dcf5d6ab12 100644
--- a/utils/tsim/src/simSystem.c
+++ b/utils/tsim/src/simSystem.c
@@ -28,7 +28,7 @@ extern bool simExecSuccess;
int32_t simInitCfg() {
taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, 1);
- taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1, true);
+ taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1);
SConfig *pCfg = taosGetCfg();
tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX);