diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md
index e2aff7a878..1f340cab30 100644
--- a/docs/en/12-taos-sql/24-show.md
+++ b/docs/en/12-taos-sql/24-show.md
@@ -189,7 +189,7 @@ show table distributed d0\G;
Show Example
*************************** 1.row ***************************
-_block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
+_block_dist: Total_Blocks=[5] Total_Size=[93.65 KB] Average_size=[18.73 KB] Compression_Ratio=[23.98 %]
Total_Blocks : Table `d0` contains total 5 blocks
diff --git a/docs/examples/rust/nativeexample/examples/subscribe_demo.rs b/docs/examples/rust/nativeexample/examples/subscribe_demo.rs
index 7551ad46b1..d54bb60e93 100644
--- a/docs/examples/rust/nativeexample/examples/subscribe_demo.rs
+++ b/docs/examples/rust/nativeexample/examples/subscribe_demo.rs
@@ -45,7 +45,7 @@ async fn main() -> anyhow::Result<()> {
taos.exec_many([
format!("DROP TOPIC IF EXISTS tmq_meters"),
format!("DROP DATABASE IF EXISTS `{db}`"),
- format!("CREATE DATABASE `{db}`"),
+ format!("CREATE DATABASE `{db}` WAL_RETENTION_PERIOD 3600"),
format!("USE `{db}`"),
// create super table
format!("CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(24))"),
diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md
index c85efa2376..ab29a1ee50 100644
--- a/docs/zh/12-taos-sql/24-show.md
+++ b/docs/zh/12-taos-sql/24-show.md
@@ -189,7 +189,7 @@ SHOW TABLE DISTRIBUTED table_name;
*************************** 1.row ***************************
-_block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
+_block_dist: Total_Blocks=[5] Total_Size=[93.65 KB] Average_size=[18.73 KB] Compression_Ratio=[23.98 %]
Total_Blocks: 表 d0 占用的 block 个数为 5 个
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 8b4d5f16df..2bc67e439f 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -212,14 +212,6 @@ enum {
FETCH_TYPE__NONE,
};
-typedef struct {
- int8_t fetchType;
- union {
- SSDataBlock data;
- void* meta;
- };
-} SFetchRet;
-
typedef struct SVarColAttr {
int32_t* offset; // start position for each entry in the list
uint32_t length; // used buffer size that contain the valid data
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index dc997221e8..1ea9714bf9 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -415,7 +415,7 @@ static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* p
return pSW;
}
-static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) {
+static FORCE_INLINE void tDeleteSchemaWrapper(SSchemaWrapper* pSchemaWrapper) {
if (pSchemaWrapper) {
taosMemoryFree(pSchemaWrapper->pSchema);
taosMemoryFree(pSchemaWrapper);
@@ -3421,10 +3421,10 @@ typedef struct {
char data[]; // SSubmitReq2
} SSubmitReq2Msg;
-int32_t tEncodeSSubmitReq2(SEncoder* pCoder, const SSubmitReq2* pReq);
-int32_t tDecodeSSubmitReq2(SDecoder* pCoder, SSubmitReq2* pReq);
-void tDestroySSubmitTbData(SSubmitTbData* pTbData, int32_t flag);
-void tDestroySSubmitReq(SSubmitReq2* pReq, int32_t flag);
+int32_t tEncodeSubmitReq(SEncoder* pCoder, const SSubmitReq2* pReq);
+int32_t tDecodeSubmitReq(SDecoder* pCoder, SSubmitReq2* pReq);
+void tDestroySubmitTbData(SSubmitTbData* pTbData, int32_t flag);
+void tDestroySubmitReq(SSubmitReq2* pReq, int32_t flag);
typedef struct {
int32_t affectedRows;
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index 61eca6cc4f..b7e6c42e3b 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -190,9 +190,9 @@ STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int
SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo);
-int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType);
+void verifyOffset(void *pWalReader, STqOffsetVal* pOffset);
-int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit);
+int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType);
void qStreamSetOpen(qTaskInfo_t tinfo);
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 48c15e9117..5fd9a8b12b 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -340,6 +340,7 @@ typedef struct SStreamMeta {
TTB* pTaskDb;
TTB* pCheckpointDb;
SHashObj* pTasks;
+ SArray* pTaskList; // SArray
void* ahandle;
TXN* txn;
FTaskExpand* expandFunc;
diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h
index b51289de5e..46dc179295 100644
--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@@ -132,7 +132,7 @@ typedef struct {
} SWalRef;
typedef struct {
- int8_t scanUncommited;
+// int8_t scanUncommited;
int8_t scanNotApplied;
int8_t scanMeta;
int8_t enableRef;
@@ -147,8 +147,6 @@ typedef struct SWalReader {
int64_t curFileFirstVer;
int64_t curVersion;
int64_t capacity;
-// int8_t curInvalid;
-// int8_t curStopped;
TdThreadMutex mutex;
SWalFilterCond cond;
// TODO remove it
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index ab89466a19..80630c1358 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -146,7 +146,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSC_CONN_KILLED TAOS_DEF_ERROR_CODE(0, 0x0215)
#define TSDB_CODE_TSC_SQL_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x0216)
#define TSDB_CODE_TSC_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0217)
-#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218)
+//#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218)
#define TSDB_CODE_TSC_EXCEED_SQL_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0219)
#define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A)
#define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B)
@@ -261,6 +261,7 @@ int32_t* taosGetErrno();
// #define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) // 2.x
#define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E)
#define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F)
+#define TSDB_CODE_MND_FIELD_VALUE_OVERFLOW TAOS_DEF_ERROR_CODE(0, 0x0370)
// mnode-func
diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h
index 3982c0d9aa..b20fc6f57a 100644
--- a/source/client/inc/clientSml.h
+++ b/source/client/inc/clientSml.h
@@ -169,6 +169,7 @@ typedef struct {
int32_t uid; // used for automatic create child table
SHashObj *childTables;
+ SHashObj *tableUids;
SHashObj *superTables;
SHashObj *pVgHash;
@@ -242,6 +243,7 @@ int8_t smlGetTsTypeByLen(int32_t len);
SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen);
SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat);
int32_t smlSetCTableName(SSmlTableInfo *oneTable);
+void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo);
STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen);
int32_t is_same_child_table_telnet(const void *a, const void *b);
int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len);
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 2ebc8e7379..fd70598efb 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -191,7 +191,7 @@ void taos_free_result(TAOS_RES *res) {
taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
taosArrayDestroy(pRsp->rsp.blockDataLen);
taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree);
- taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
+ taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSchemaWrapper);
// taosx
taosArrayDestroy(pRsp->rsp.createTableLen);
taosArrayDestroyP(pRsp->rsp.createTableReq, taosMemoryFree);
@@ -204,7 +204,7 @@ void taos_free_result(TAOS_RES *res) {
taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
taosArrayDestroy(pRsp->rsp.blockDataLen);
taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree);
- taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
+ taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSchemaWrapper);
pRsp->resInfo.pRspMsg = NULL;
doFreeReqResultInfo(&pRsp->resInfo);
taosMemoryFree(pRsp);
diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 7267c72ec2..ce06e0eac4 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -195,6 +195,20 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable) {
return TSDB_CODE_SUCCESS;
}
+void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo){
+ char key[TSDB_TABLE_NAME_LEN * 2 + 1] = {0};
+ size_t nLen = strlen(tinfo->childTableName);
+ memcpy(key, currElement->measure, currElement->measureLen);
+ memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen);
+ void *uid = taosHashGet(info->tableUids, key, currElement->measureLen + 1 + nLen); // use \0 as separator for stable name and child table name
+ if (uid == NULL) {
+ tinfo->uid = info->uid++;
+ taosHashPut(info->tableUids, key, currElement->measureLen + 1 + nLen, &tinfo->uid, sizeof(uint64_t));
+ }else{
+ tinfo->uid = *(uint64_t*)uid;
+ }
+}
+
SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) {
SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1);
if (!meta) {
@@ -558,10 +572,15 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
return 0;
}
+#define BOUNDARY 1024
static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
int32_t result = 1;
- while (result <= length) {
- result *= 2;
+ if (length >= BOUNDARY){
+ result = length;
+ }else{
+ while (result <= length) {
+ result *= 2;
+ }
}
if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE;
@@ -657,7 +676,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
len += field->bytes;
}
if(len > maxLen){
- return TSDB_CODE_TSC_INVALID_VALUE;
+ return isTag ? TSDB_CODE_PAR_INVALID_TAGS_LENGTH : TSDB_CODE_PAR_INVALID_ROW_LENGTH;
}
return TSDB_CODE_SUCCESS;
@@ -1137,6 +1156,7 @@ void smlDestroyInfo(SSmlHandle *info) {
taosHashCleanup(info->pVgHash);
taosHashCleanup(info->childTables);
taosHashCleanup(info->superTables);
+ taosHashCleanup(info->tableUids);
for (int i = 0; i < taosArrayGetSize(info->tagJsonArray); i++) {
cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i);
@@ -1187,6 +1207,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+ info->tableUids = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
info->id = smlGenId();
@@ -1197,7 +1218,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
info->valueJsonArray = taosArrayInit(8, POINTER_BYTES);
info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv));
- if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables) {
+ if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables || NULL == info->tableUids) {
uError("create SSmlHandle failed");
goto cleanup;
}
@@ -1315,23 +1336,23 @@ static int32_t smlInsertData(SSmlHandle *info) {
if (info->pRequest->dbList == NULL) {
info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN);
}
- void *data = taosArrayReserve(info->pRequest->dbList, 1);
- memcpy(data, info->pRequest->pDb,
- TSDB_DB_FNAME_LEN > strlen(info->pRequest->pDb) ? strlen(info->pRequest->pDb) : TSDB_DB_FNAME_LEN);
+ char *data = (char*)taosArrayReserve(info->pRequest->dbList, 1);
+ SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
+ tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
+ tNameGetFullDbName(&pName, data);
SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
while (oneTable) {
SSmlTableInfo *tableData = *oneTable;
-
- SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
- tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
- memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName));
+ tstrncpy(pName.tname, tableData->sTableName, tableData->sTableNameLen + 1);
if (info->pRequest->tableList == NULL) {
info->pRequest->tableList = taosArrayInit(1, sizeof(SName));
}
taosArrayPush(info->pRequest->tableList, &pName);
+ tstrncpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName) + 1);
+
SRequestConnInfo conn = {0};
conn.pTrans = info->taos->pAppInfo->pTransporter;
conn.requestId = info->pRequest->requestId;
@@ -1423,6 +1444,7 @@ int32_t smlClearForRerun(SSmlHandle *info) {
taosHashClear(info->childTables);
taosHashClear(info->superTables);
+ taosHashClear(info->tableUids);
if (!info->dataFormat) {
if (unlikely(info->lines != NULL)) {
@@ -1557,7 +1579,10 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
do {
code = smlModifyDBSchemas(info);
if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA || code == TSDB_CODE_PAR_TOO_MANY_COLUMNS
- || code == TSDB_CODE_PAR_INVALID_TAGS_NUM) break;
+ || code == TSDB_CODE_PAR_INVALID_TAGS_NUM || code == TSDB_CODE_PAR_INVALID_TAGS_LENGTH
+ || code == TSDB_CODE_PAR_INVALID_ROW_LENGTH || code == TSDB_CODE_MND_FIELD_VALUE_OVERFLOW) {
+ break;
+ }
taosMsleep(100);
uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum);
} while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES);
@@ -1642,7 +1667,8 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
info->cost.endTime = taosGetTimestampUs();
info->cost.code = code;
if (code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING ||
- code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT) {
+ code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT ||
+ code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
if (cnt++ >= 10) {
uInfo("SML:%" PRIx64 " retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
break;
diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c
index b0ae316031..7ccf930964 100644
--- a/source/client/src/clientSmlJson.c
+++ b/source/client/src/clientSmlJson.c
@@ -778,7 +778,7 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
tinfo->tags = taosArrayDup(preLineKV, NULL);
smlSetCTableName(tinfo);
- tinfo->uid = info->uid++;
+ getTableUid(info, elements, tinfo);
if (info->dataFormat) {
info->currSTableMeta->uid = tinfo->uid;
tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c
index 1732473c11..2f7e8a0f97 100644
--- a/source/client/src/clientSmlLine.c
+++ b/source/client/src/clientSmlLine.c
@@ -312,7 +312,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
}
smlSetCTableName(tinfo);
- tinfo->uid = info->uid++;
+ getTableUid(info, currElement, tinfo);
if (info->dataFormat) {
info->currSTableMeta->uid = tinfo->uid;
tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c
index 036442573d..c5dd20ba7b 100644
--- a/source/client/src/clientSmlTelnet.c
+++ b/source/client/src/clientSmlTelnet.c
@@ -206,7 +206,7 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
tinfo->tags = taosArrayDup(preLineKV, NULL);
smlSetCTableName(tinfo);
- tinfo->uid = info->uid++;
+ getTableUid(info, elements, tinfo);
if (info->dataFormat) {
info->currSTableMeta->uid = tinfo->uid;
tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c
index 6e529f1a0b..975b304bf4 100644
--- a/source/client/src/clientStmt.c
+++ b/source/client/src/clientStmt.c
@@ -325,7 +325,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) {
taosHashCleanup(pStmt->exec.pBlockHash);
pStmt->exec.pBlockHash = NULL;
- tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE);
taosMemoryFreeClear(pStmt->exec.pCurrTbData);
STMT_ERR_RET(stmtCleanBindInfo(pStmt));
@@ -895,7 +895,7 @@ int stmtExec(TAOS_STMT* stmt) {
if (STMT_TYPE_QUERY == pStmt->sql.type) {
launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL);
} else {
- tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE);
taosMemoryFreeClear(pStmt->exec.pCurrTbData);
STMT_ERR_RET(qCloneCurrentTbData(pStmt->exec.pCurrBlock, &pStmt->exec.pCurrTbData));
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index b5ae9116ef..b488af9ba1 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -864,7 +864,7 @@ static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
taosArrayDestroyP(pRsp->dataRsp.blockData, taosMemoryFree);
taosArrayDestroy(pRsp->dataRsp.blockDataLen);
taosArrayDestroyP(pRsp->dataRsp.blockTbName, taosMemoryFree);
- taosArrayDestroyP(pRsp->dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
+ taosArrayDestroyP(pRsp->dataRsp.blockSchema, (FDelete)tDeleteSchemaWrapper);
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper;
taosMemoryFreeClear(pRsp->pEpset);
@@ -877,7 +877,7 @@ static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) {
taosArrayDestroyP(pRsp->taosxRsp.blockData, taosMemoryFree);
taosArrayDestroy(pRsp->taosxRsp.blockDataLen);
taosArrayDestroyP(pRsp->taosxRsp.blockTbName, taosMemoryFree);
- taosArrayDestroyP(pRsp->taosxRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
+ taosArrayDestroyP(pRsp->taosxRsp.blockSchema, (FDelete)tDeleteSchemaWrapper);
// taosx
taosArrayDestroy(pRsp->taosxRsp.createTableLen);
taosArrayDestroyP(pRsp->taosxRsp.createTableReq, taosMemoryFree);
diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp
index b9062fc8ff..56f68e5972 100644
--- a/source/client/test/clientTests.cpp
+++ b/source/client/test/clientTests.cpp
@@ -1053,9 +1053,9 @@ TEST(clientCase, sub_db_test) {
}
TEST(clientCase, sub_tb_test) {
- taos_options(TSDB_OPTION_CONFIGDIR, "/home/tests/dir/cfg/");
+ taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg");
- TAOS* pConn = taos_connect("vm116", "root", "taosdata", NULL, 0);
+ TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr);
tmq_conf_t* conf = tmq_conf_new();
@@ -1091,7 +1091,7 @@ TEST(clientCase, sub_tb_test) {
int32_t precision = 0;
int32_t totalRows = 0;
int32_t msgCnt = 0;
- int32_t timeout = 25000;
+ int32_t timeout = 2500000;
int32_t count = 0;
@@ -1117,10 +1117,10 @@ TEST(clientCase, sub_tb_test) {
fields = taos_fetch_fields(pRes);
numOfFields = taos_field_count(pRes);
totalRows += 1;
- if (totalRows % 100000 == 0) {
+// if (totalRows % 100000 == 0) {
taos_print_row(buf, row, fields, numOfFields);
printf("row content: %s\n", buf);
- }
+// }
}
taos_free_result(pRes);
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 3558feaa66..5f7e43668a 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -120,6 +120,7 @@ int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const
pColumnInfoData->varmeta.length += dataLen;
} else {
memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * rowIndex, pData, pColumnInfoData->info.bytes);
+ colDataClearNull_f(pColumnInfoData->nullbitmap, rowIndex);
}
return 0;
@@ -1949,12 +1950,11 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
}
}
}
-
#endif
// for debug
char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) {
- int32_t size = 2048;
+ int32_t size = 2048*1024;
*pDataBuf = taosMemoryCalloc(size, 1);
char* dumpBuf = *pDataBuf;
char pBuf[128] = {0};
@@ -1970,7 +1970,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
if (len >= size - 1) return dumpBuf;
for (int32_t j = 0; j < rows; j++) {
- len += snprintf(dumpBuf + len, size - len, "%s |", flag);
+ len += snprintf(dumpBuf + len, size - len, "%s %d|", flag, j);
if (len >= size - 1) return dumpBuf;
for (int32_t k = 0; k < colNum; k++) {
@@ -2374,7 +2374,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat
}
SRow* pRow = NULL;
if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) {
- tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
goto _end;
}
ASSERT(pRow);
@@ -2388,7 +2388,7 @@ _end:
if (terrno != 0) {
*ppReq = NULL;
if (pReq) {
- tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFreeClear(pReq);
}
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index f379084cf5..b18bd882ae 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -1509,7 +1509,9 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) {
// STSchema ========================================
STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) {
STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols);
- if (pTSchema == NULL) return NULL;
+ if (pTSchema == NULL) {
+ return NULL;
+ }
pTSchema->numOfCols = numOfCols;
pTSchema->version = version;
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 8b14b7fbe7..7102e556cc 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -7058,7 +7058,7 @@ void tDeleteSMqDataRsp(SMqDataRsp *pRsp) {
pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen);
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
pRsp->blockData = NULL;
- taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);
+ taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper);
pRsp->blockSchema = NULL;
taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
pRsp->blockTbName = NULL;
@@ -7159,7 +7159,7 @@ void tDeleteSTaosxRsp(STaosxRsp *pRsp) {
pRsp->blockDataLen = NULL;
taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
pRsp->blockData = NULL;
- taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);
+ taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper);
pRsp->blockSchema = NULL;
taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
pRsp->blockTbName = NULL;
@@ -7332,7 +7332,7 @@ _exit:
return 0;
}
-int32_t tEncodeSSubmitReq2(SEncoder *pCoder, const SSubmitReq2 *pReq) {
+int32_t tEncodeSubmitReq(SEncoder *pCoder, const SSubmitReq2 *pReq) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeU64v(pCoder, taosArrayGetSize(pReq->aSubmitTbData)) < 0) return -1;
@@ -7344,7 +7344,7 @@ int32_t tEncodeSSubmitReq2(SEncoder *pCoder, const SSubmitReq2 *pReq) {
return 0;
}
-int32_t tDecodeSSubmitReq2(SDecoder *pCoder, SSubmitReq2 *pReq) {
+int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) {
int32_t code = 0;
memset(pReq, 0, sizeof(*pReq));
@@ -7387,7 +7387,7 @@ _exit:
return code;
}
-void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
+void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
if (NULL == pTbData) {
return;
}
@@ -7433,14 +7433,14 @@ void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
}
}
-void tDestroySSubmitReq(SSubmitReq2 *pReq, int32_t flag) {
+void tDestroySubmitReq(SSubmitReq2 *pReq, int32_t flag) {
if (pReq->aSubmitTbData == NULL) return;
int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData);
SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData);
for (int32_t i = 0; i < nSubmitTbData; i++) {
- tDestroySSubmitTbData(&aSubmitTbData[i], flag);
+ tDestroySubmitTbData(&aSubmitTbData[i], flag);
}
taosArrayDestroy(pReq->aSubmitTbData);
pReq->aSubmitTbData = NULL;
diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h
index 0229735952..94c4eae83f 100644
--- a/source/dnode/mnode/impl/inc/mndVgroup.h
+++ b/source/dnode/mnode/impl/inc/mndVgroup.h
@@ -50,6 +50,8 @@ void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *p
void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid);
+int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup);
+
#ifdef __cplusplus
}
#endif
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 63bcef2a5b..8b708c3e0f 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -797,6 +797,11 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
return -1;
}
+ if(pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags){
+ terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
+ return -1;
+ }
+
for (int32_t i = 0; i < pDst->numOfColumns; ++i) {
SField *pField = taosArrayGet(pCreate->pColumns, i);
SSchema *pSchema = &pDst->pColumns[i];
@@ -927,6 +932,11 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
return -1;
}
+ if(pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags){
+ terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
+ return -1;
+ }
+
for (int32_t i = 0; i < pDst->numOfColumns; ++i) {
SField *pField = taosArrayGet(createReq->pColumns, i);
SSchema *pSchema = &pDst->pColumns[i];
@@ -1153,6 +1163,11 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p
if (mndAllocStbSchemas(pOld, pNew) != 0) {
return -1;
}
+
+ if(pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ntags){
+ terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
+ return -1;
+ }
for (int32_t i = 0; i < ntags; i++) {
SField *pField = taosArrayGet(pFields, i);
@@ -1461,6 +1476,11 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
return -1;
}
+ if(pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ncols){
+ terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
+ return -1;
+ }
+
for (int32_t i = 0; i < ncols; i++) {
SField *pField = taosArrayGet(pFields, i);
if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) {
@@ -2524,6 +2544,9 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t
if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) {
metaRsp.numOfColumns = -1;
metaRsp.suid = pStbVersion->suid;
+ tstrncpy(metaRsp.dbFName, pStbVersion->dbFName, sizeof(metaRsp.dbFName));
+ tstrncpy(metaRsp.tbName, pStbVersion->stbName, sizeof(metaRsp.tbName));
+ tstrncpy(metaRsp.stbName, pStbVersion->stbName, sizeof(metaRsp.stbName));
taosArrayPush(hbRsp.pMetaRsp, &metaRsp);
continue;
}
diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c
index 573c60549e..e62102fa77 100644
--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@@ -133,10 +133,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub,
const SMqRebOutputVg *pRebVg) {
- if (pRebVg->oldConsumerId == pRebVg->newConsumerId) {
- terrno = TSDB_CODE_MND_INVALID_SUB_OPTION;
- return -1;
- }
+// if (pRebVg->oldConsumerId == pRebVg->newConsumerId) {
+// terrno = TSDB_CODE_MND_INVALID_SUB_OPTION;
+// return -1;
+// }
void *buf;
int32_t tlen;
@@ -197,7 +197,7 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
return pRebSub;
}
-static void doRemoveExistedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) {
+static void doRemoveLostConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) {
int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
const char *pSubKey = pOutput->pSub->key;
@@ -269,6 +269,18 @@ static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) {
}
}
+static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){
+ for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){
+ SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i);
+ SMqRebOutputVg outputVg = {
+ .oldConsumerId = pConsumerEp->consumerId,
+ .newConsumerId = pConsumerEp->consumerId,
+ .pVgEp = pVgEp,
+ };
+ taosArrayPush(pOutput->rebVgs, &outputVg);
+ }
+}
+
static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt,
int32_t imbConsumerNum) {
const char *pSubKey = pOutput->pSub->key;
@@ -290,24 +302,19 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas
taosArrayPush(pOutput->modifyConsumers, &pConsumerEp->consumerId);
if (consumerVgNum > minVgCnt) {
if (imbCnt < imbConsumerNum) {
- if (consumerVgNum == minVgCnt + 1) {
- imbCnt++;
- continue;
- } else {
- // pop until equal minVg + 1
- while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) {
- SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
- SMqRebOutputVg outputVg = {
- .oldConsumerId = pConsumerEp->consumerId,
- .newConsumerId = -1,
- .pVgEp = pVgEp,
- };
- taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
- mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId,
- pConsumerEp->consumerId);
- }
- imbCnt++;
+ // pop until equal minVg + 1
+ while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) {
+ SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
+ SMqRebOutputVg outputVg = {
+ .oldConsumerId = pConsumerEp->consumerId,
+ .newConsumerId = -1,
+ .pVgEp = pVgEp,
+ };
+ taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
+ mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId,
+ pConsumerEp->consumerId);
}
+ imbCnt++;
} else {
// all the remain consumers should only have the number of vgroups, which is equalled to the value of minVg
while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt) {
@@ -323,6 +330,7 @@ static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHas
}
}
}
+ putNoTransferToOutput(pOutput, pConsumerEp);
}
}
@@ -339,7 +347,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
// 2. check and get actual removed consumers, put their vg into pHash
- doRemoveExistedConsumers(pOutput, pHash, pInput);
+ doRemoveLostConsumers(pOutput, pHash, pInput);
// 3. if previously no consumer, there are vgs not assigned, put these vg into pHash
addUnassignedVgroups(pOutput, pHash);
diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c
index 523753d7c6..2a0d753722 100644
--- a/source/dnode/mnode/impl/src/mndUser.c
+++ b/source/dnode/mnode/impl/src/mndUser.c
@@ -236,7 +236,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
SDB_SET_INT32(pRaw, dataPos, *useDb, _OVER)
- useDb = taosHashIterate(pUser->writeTbs, useDb);
+ useDb = taosHashIterate(pUser->useDbs, useDb);
}
SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index ed1fddb63f..0003d07fd6 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -2006,7 +2006,7 @@ static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans,
return 0;
}
-static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) {
+int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) {
int32_t code = -1;
STrans *pTrans = NULL;
SSdbRaw *pRaw = NULL;
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index cefc4fa63e..7352bbc0fe 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -153,11 +153,15 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
// 2.save task
+ taosWLockLatch(&pSnode->pMeta->lock);
code = streamMetaAddDeployedTask(pSnode->pMeta, -1, pTask);
if (code < 0) {
+ taosWUnLockLatch(&pSnode->pMeta->lock);
return -1;
}
+ taosWUnLockLatch(&pSnode->pMeta->lock);
+
// 3.go through recover steps to fill history
if (pTask->fillHistory) {
streamSetParamForRecover(pTask);
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 002dcda488..88460cd3ca 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -231,7 +231,7 @@ typedef struct SSnapContext {
} SSnapContext;
typedef struct STqReader {
- SPackedData msg2;
+ SPackedData msg;
SSubmitReq2 submit;
int32_t nextBlk;
int64_t lastBlkUid;
@@ -241,8 +241,9 @@ typedef struct STqReader {
SArray *pColIdList; // SArray
int32_t cachedSchemaVer;
int64_t cachedSchemaSuid;
+ int64_t cachedSchemaUid;
SSchemaWrapper *pSchemaWrapper;
- STSchema *pSchema;
+ SSDataBlock *pResBlock;
} STqReader;
STqReader *tqReaderOpen(SVnode *pVnode);
@@ -254,13 +255,13 @@ int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList);
int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id);
-int32_t tqNextBlock(STqReader *pReader, SSDataBlock* pBlock);
-int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData);
-
-int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
+int32_t tqNextBlockInWal(STqReader* pReader);
bool tqNextBlockImpl(STqReader *pReader);
+
+int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData);
+int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
-int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet);
+int32_t tqRetrieveDataBlock(STqReader *pReader, SSubmitTbData **pSubmitTbDataRet);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 7668d45108..1aea479511 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -193,7 +193,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode);
void tqNotifyClose(STQ*);
void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
-int tqRegisterPushEntry(STQ* pTq, void* handle, SRpcMsg* pMsg);
+int tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg);
int tqUnregisterPushHandle(STQ* pTq, void* pHandle);
int tqStartStreamTasks(STQ* pTq); // restore all stream tasks after vnode launching completed.
@@ -212,7 +212,6 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg
int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
-int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit);
int32_t tqProcessSubmitReqForSubscribe(STQ* pTq);
int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver);
int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg);
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index 2359a165b7..d464f64de3 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -639,7 +639,6 @@ tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) {
STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
STSchema *pTSchema = NULL;
SSchemaWrapper *pSW = NULL;
- SSchema *pSchema = NULL;
pSW = metaGetTableSchema(pMeta, uid, sver, lock);
if (!pSW) return NULL;
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 56e802d4fb..707dd66e30 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -217,8 +217,8 @@ typedef struct STableInfoForChildTable {
static void destroySTableInfoForChildTable(void* data) {
STableInfoForChildTable* pData = (STableInfoForChildTable*)data;
taosMemoryFree(pData->tableName);
- tDeleteSSchemaWrapper(pData->schemaRow);
- tDeleteSSchemaWrapper(pData->tagRow);
+ tDeleteSchemaWrapper(pData->schemaRow);
+ tDeleteSchemaWrapper(pData->tagRow);
}
static void MoveToSnapShotVersion(SSnapContext* ctx) {
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 96eec89127..83f2ece571 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -673,8 +673,8 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
metaUpdateUidIdx(pMeta, &nStbEntry);
metaULock(pMeta);
- tDeleteSSchemaWrapper(tag);
- tDeleteSSchemaWrapper(row);
+ tDeleteSchemaWrapper(tag);
+ tDeleteSchemaWrapper(row);
if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
tDecoderClear(&dc);
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 69b3f9c3e0..20e04f122b 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -684,7 +684,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
}
if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) {
- tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq);
smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8
" failed since %s",
@@ -696,7 +696,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version);
if (pReq) {
- tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq);
}
}
diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c
index 2a26f65bf9..6a4bddc991 100644
--- a/source/dnode/vnode/src/sma/smaTimeRange.c
+++ b/source/dnode/vnode/src/sma/smaTimeRange.c
@@ -299,7 +299,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
}
SRow *pRow = NULL;
if ((terrno = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) {
- tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
goto _end;
}
taosArrayPush(tbData.aRowP, &pRow);
@@ -309,7 +309,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
}
// encode
- tEncodeSize(tEncodeSSubmitReq2, pReq, len, terrno);
+ tEncodeSize(tEncodeSubmitReq, pReq, len, terrno);
if (TSDB_CODE_SUCCESS == terrno) {
SEncoder encoder;
len += sizeof(SSubmitReq2Msg);
@@ -321,7 +321,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
((SSubmitReq2Msg *)pBuf)->header.contLen = htonl(len);
((SSubmitReq2Msg *)pBuf)->version = htobe64(1);
tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg));
- if (tEncodeSSubmitReq2(&encoder, pReq) < 0) {
+ if (tEncodeSubmitReq(&encoder, pReq) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
/*vError("failed to encode submit req since %s", terrstr());*/
}
@@ -332,7 +332,7 @@ _end:
taosArrayDestroy(tagArray);
taosArrayDestroy(pVals);
if (pReq) {
- tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq);
}
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 12b81b6c3f..311d637be8 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -445,6 +445,7 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t
}
int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) {
+ int ret = 0;
SMqRebVgReq req = {0};
tDecodeSMqRebVgReq(msg, &req);
@@ -463,8 +464,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
if (req.newConsumerId == -1) {
tqError("vgId:%d, tq invalid re-balance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId);
- taosMemoryFree(req.qmsg);
- return 0;
+ goto end;
}
STqHandle tqHandle = {0};
@@ -481,8 +481,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
// TODO version should be assigned and refed during preprocess
SWalRef* pRef = walRefCommittedVer(pVnode->pWal);
if (pRef == NULL) {
- taosMemoryFree(req.qmsg);
- return -1;
+ ret = -1;
+ goto end;
}
int64_t ver = pRef->refVer;
@@ -534,49 +534,42 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
tqDebug("try to persist handle %s consumer:0x%" PRIx64 " , old consumer:0x%" PRIx64, req.subKey,
pHandle->consumerId, oldConsumerId);
- if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
- taosMemoryFree(req.qmsg);
- return -1;
- }
+ ret = tqMetaSaveHandle(pTq, req.subKey, pHandle);
+ goto end;
} else {
if (pHandle->consumerId == req.newConsumerId) { // do nothing
tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId);
- atomic_store_32(&pHandle->epoch, -1);
atomic_add_fetch_32(&pHandle->epoch, 1);
- taosMemoryFree(req.qmsg);
- return tqMetaSaveHandle(pTq, req.subKey, pHandle);
+
} else {
tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
req.newConsumerId);
-
- // kill executing task
- qTaskInfo_t pTaskInfo = pHandle->execHandle.task;
- if (pTaskInfo != NULL) {
- qKillTask(pTaskInfo, TSDB_CODE_SUCCESS);
- }
-
- taosWLockLatch(&pTq->lock);
- atomic_store_32(&pHandle->epoch, 0);
-
- // remove if it has been register in the push manager, and return one empty block to consumer
- tqUnregisterPushHandle(pTq, pHandle);
-
atomic_store_64(&pHandle->consumerId, req.newConsumerId);
-
- if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
- qStreamCloseTsdbReader(pTaskInfo);
- }
-
- taosWUnLockLatch(&pTq->lock);
- if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
- taosMemoryFree(req.qmsg);
- return -1;
- }
+ atomic_store_32(&pHandle->epoch, 0);
}
+ // kill executing task
+ qTaskInfo_t pTaskInfo = pHandle->execHandle.task;
+ if (pTaskInfo != NULL) {
+ qKillTask(pTaskInfo, TSDB_CODE_SUCCESS);
+ }
+
+ taosWLockLatch(&pTq->lock);
+ // remove if it has been register in the push manager, and return one empty block to consumer
+ tqUnregisterPushHandle(pTq, pHandle);
+
+
+ if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+ qStreamCloseTsdbReader(pTaskInfo);
+ }
+
+ taosWUnLockLatch(&pTq->lock);
+ ret = tqMetaSaveHandle(pTq, req.subKey, pHandle);
+ goto end;
}
+end:
taosMemoryFree(req.qmsg);
- return 0;
+ return ret;
}
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
@@ -781,13 +774,17 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
tDecoderClear(&decoder);
// 2.save task, use the newest commit version as the initial start version of stream task.
+ taosWLockLatch(&pTq->pStreamMeta->lock);
code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask);
if (code < 0) {
tqError("vgId:%d failed to add s-task:%s, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr,
streamMetaGetNumOfTasks(pTq->pStreamMeta));
+ taosWUnLockLatch(&pTq->pStreamMeta->lock);
return -1;
}
+ taosWUnLockLatch(&pTq->pStreamMeta->lock);
+
// 3.go through recover steps to fill history
if (pTask->fillHistory) {
streamTaskCheckDownstream(pTask, sversion);
@@ -1069,12 +1066,15 @@ int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode);
taosWLockLatch(&pTq->lock);
- if(taosHashGetSize(pTq->pPushMgr) > 0){
- void *pIter = taosHashIterate(pTq->pPushMgr, NULL);
- while(pIter){
+
+ if (taosHashGetSize(pTq->pPushMgr) > 0) {
+ void* pIter = taosHashIterate(pTq->pPushMgr, NULL);
+
+ while (pIter) {
STqHandle* pHandle = *(STqHandle**)pIter;
- tqDebug("vgId:%d start set submit for pHandle:%p, consume id:0x%"PRIx64, vgId, pHandle, pHandle->consumerId);
- if(ASSERT(pHandle->msg != NULL)){
+ tqDebug("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId);
+
+ if (ASSERT(pHandle->msg != NULL)) {
tqError("pHandle->msg should not be null");
break;
}else{
@@ -1083,77 +1083,15 @@ int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) {
taosMemoryFree(pHandle->msg);
pHandle->msg = NULL;
}
+
pIter = taosHashIterate(pTq->pPushMgr, pIter);
}
+
taosHashClear(pTq->pPushMgr);
}
+
// unlock
taosWUnLockLatch(&pTq->lock);
-
- return 0;
-}
-
-int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
-#if 0
- void* pIter = NULL;
- SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit, STREAM_INPUT__DATA_SUBMIT);
- if (pSubmit == NULL) {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- tqError("failed to create data submit for stream since out of memory");
- saveOffsetForAllTasks(pTq, submit.ver);
- return -1;
- }
-
- SArray* pInputQueueFullTasks = taosArrayInit(4, POINTER_BYTES);
-
- while (1) {
- pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
- if (pIter == NULL) {
- break;
- }
-
- SStreamTask* pTask = *(SStreamTask**)pIter;
- if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
- continue;
- }
-
- if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
- tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId,
- pTask->status.taskStatus);
- continue;
- }
-
- // check if offset value exists
- char key[128] = {0};
- createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
-
- if (tInputQueueIsFull(pTask)) {
- STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
-
- int64_t ver = submit.ver;
- if (pOffset == NULL) {
- doSaveTaskOffset(pTq->pOffsetStore, key, submit.ver);
- } else {
- ver = pOffset->val.version;
- }
-
- tqDebug("s-task:%s input queue is full, discard submit block, ver:%" PRId64, pTask->id.idStr, ver);
- taosArrayPush(pInputQueueFullTasks, &pTask);
- continue;
- }
-
- // check if offset value exists
- STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
- ASSERT(pOffset == NULL);
-
- addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver);
- }
-
- streamDataSubmitDestroy(pSubmit);
- taosFreeQitem(pSubmit);
-#endif
-
- tqStartStreamTasks(pTq);
return 0;
}
@@ -1323,11 +1261,12 @@ FAIL:
int32_t tqCheckLogInWal(STQ* pTq, int64_t sversion) { return sversion <= pTq->walLogLastVer; }
int32_t tqStartStreamTasks(STQ* pTq) {
- int32_t vgId = TD_VID(pTq->pVnode);
-
+ int32_t vgId = TD_VID(pTq->pVnode);
SStreamMeta* pMeta = pTq->pStreamMeta;
+
taosWLockLatch(&pMeta->lock);
- int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks);
+
+ int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
if (numOfTasks == 0) {
tqInfo("vgId:%d no stream tasks exists", vgId);
taosWUnLockLatch(&pTq->pStreamMeta->lock);
diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c
index 0575b7299d..950c5ea96b 100644
--- a/source/dnode/vnode/src/tq/tqPush.c
+++ b/source/dnode/vnode/src/tq/tqPush.c
@@ -16,266 +16,26 @@
#include "tq.h"
#include "vnd.h"
-#if 0
-void tqTmrRspFunc(void* param, void* tmrId) {
- STqHandle* pHandle = (STqHandle*)param;
- atomic_store_8(&pHandle->pushHandle.tmrStopped, 1);
-}
-
-static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataRsp* pRsp) {
- SStreamDataSubmit* pSubmit = *ppSubmit;
- while (pSubmit != NULL) {
- if (tqLogScanExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) {
- }
- // update processed
- atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver);
- streamQueueProcessSuccess(&pHandle->pushHandle.inputQ);
- streamDataSubmitDestroy(pSubmit);
- if (pRsp->blockNum > 0) {
- *ppSubmit = pSubmit;
- return 0;
- } else {
- pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ);
- }
- }
- *ppSubmit = pSubmit;
- return -1;
-}
-
-int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) {
- SMqDataRsp rsp = {0};
- // 1. guard and set status executing
- int8_t execStatus = atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE,
- TASK_EXEC_STATUS__EXECUTING);
- if (execStatus == TASK_EXEC_STATUS__IDLE) {
- SStreamDataSubmit* pSubmit = NULL;
- // 2. check processedVer
- // 2.1. if not missed, get msg from queue
- // 2.2. if missed, scan wal
- pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ);
- while (pHandle->pushHandle.processedVer <= pSubmit->ver) {
- // read from wal
- }
- while (pHandle->pushHandle.processedVer > pSubmit->ver + 1) {
- streamQueueProcessSuccess(&pHandle->pushHandle.inputQ);
- streamDataSubmitDestroy(pSubmit);
- pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ);
- if (pSubmit == NULL) break;
- }
- // 3. exec, after each success, update processed ver
- // first run
- if (tqLoopExecFromQueue(pTq, pHandle, &pSubmit, &rsp) == 0) {
- goto SEND_RSP;
- }
- // set exec status closing
- atomic_store_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__CLOSING);
- // second run
- if (tqLoopExecFromQueue(pTq, pHandle, &pSubmit, &rsp) == 0) {
- goto SEND_RSP;
- }
- // set exec status idle
- atomic_store_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE);
- }
-SEND_RSP:
- // 4. if get result
- // 4.1 set exec input status blocked and exec status idle
- atomic_store_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE);
- // 4.2 rpc send
- rsp.rspOffset = pHandle->pushHandle.processedVer;
- /*if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) {*/
- /*return -1;*/
- /*}*/
- // 4.3 clear rpc info
- memset(&pHandle->pushHandle.rpcInfo, 0, sizeof(SRpcHandleInfo));
- return 0;
-}
-
-int32_t tqOpenPushHandle(STQ* pTq, STqHandle* pHandle) {
- memset(&pHandle->pushHandle, 0, sizeof(STqPushHandle));
- pHandle->pushHandle.inputQ.queue = taosOpenQueue();
- pHandle->pushHandle.inputQ.qall = taosAllocateQall();
- if (pHandle->pushHandle.inputQ.queue == NULL || pHandle->pushHandle.inputQ.qall == NULL) {
- if (pHandle->pushHandle.inputQ.queue) {
- taosCloseQueue(pHandle->pushHandle.inputQ.queue);
- }
- if (pHandle->pushHandle.inputQ.qall) {
- taosFreeQall(pHandle->pushHandle.inputQ.qall);
- }
- return -1;
- }
- return 0;
-}
-
-int32_t tqPreparePush(STQ* pTq, STqHandle* pHandle, int64_t reqId, const SRpcHandleInfo* pInfo, int64_t processedVer,
- int64_t timeout) {
- memcpy(&pHandle->pushHandle.rpcInfo, pInfo, sizeof(SRpcHandleInfo));
- atomic_store_64(&pHandle->pushHandle.reqId, reqId);
- atomic_store_64(&pHandle->pushHandle.processedVer, processedVer);
- atomic_store_8(&pHandle->pushHandle.inputStatus, TASK_INPUT_STATUS__NORMAL);
- atomic_store_8(&pHandle->pushHandle.tmrStopped, 0);
- taosTmrReset(tqTmrRspFunc, (int32_t)timeout, pHandle, tqMgmt.timer, &pHandle->pushHandle.timerId);
- return 0;
-}
-
-int32_t tqEnqueue(STqHandle* pHandle, SStreamDataSubmit* pSubmit) {
- int8_t inputStatus = atomic_load_8(&pHandle->pushHandle.inputStatus);
- if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
- SStreamDataSubmit* pSubmitClone = streamSubmitBlockClone(pSubmit);
- if (pSubmitClone == NULL) {
- return -1;
- }
- taosWriteQitem(pHandle->pushHandle.inputQ.queue, pSubmitClone);
- return 0;
- }
- return -1;
-}
-
-int32_t tqSendExecReq(STQ* pTq, STqHandle* pHandle) {
- //
- return 0;
-}
-
-int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver, SRpcHandleInfo handleInfo) {
- if (msgType != TDMT_VND_SUBMIT) return 0;
- void* pIter = NULL;
- STqHandle* pHandle = NULL;
- SSubmitReq* pReq = (SSubmitReq*)msg;
- int32_t workerId = 4;
- int64_t fetchOffset = ver;
-
- while (1) {
- pIter = taosHashIterate(pTq->pushMgr, pIter);
- if (pIter == NULL) break;
- pHandle = *(STqHandle**)pIter;
-
- taosWLockLatch(&pHandle->pushHandle.lock);
-
- SMqDataRsp rsp = {0};
- rsp.reqOffset = pHandle->pushHandle.reqOffset;
- rsp.blockData = taosArrayInit(0, sizeof(void*));
- rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
-
- if (msgType == TDMT_VND_SUBMIT) {
- tqLogScanExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId);
- } else {
- tqError("tq push unexpected msg type %d", msgType);
- }
-
- if (rsp.blockNum == 0) {
- taosWUnLockLatch(&pHandle->pushHandle.lock);
- continue;
- }
-
- rsp.rspOffset = fetchOffset;
-
- int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
- void* buf = rpcMallocCont(tlen);
- if (buf == NULL) {
- // todo free
- return -1;
- }
-
- ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
- ((SMqRspHead*)buf)->epoch = pHandle->pushHandle.epoch;
- ((SMqRspHead*)buf)->consumerId = pHandle->pushHandle.consumerId;
-
- void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
- tEncodeSMqDataBlkRsp(&abuf, &rsp);
-
- SRpcMsg resp = {
- .info = pHandle->pushHandle.rpcInfo,
- .pCont = buf,
- .contLen = tlen,
- .code = 0,
- };
- tmsgSendRsp(&resp);
-
- memset(&pHandle->pushHandle.rpcInfo, 0, sizeof(SRpcHandleInfo));
- taosWUnLockLatch(&pHandle->pushHandle.lock);
-
- tqDebug("vgId:%d offset %" PRId64 " from consumer:%" PRId64 ", (epoch %d) send rsp, block num: %d, req:%" PRId64 ", rsp:%" PRId64,
- TD_VID(pTq->pVnode), fetchOffset, pHandle->pushHandle.consumerId, pHandle->pushHandle.epoch, rsp.blockNum,
- rsp.reqOffset, rsp.rspOffset);
-
- // TODO destroy
- taosArrayDestroy(rsp.blockData);
- taosArrayDestroy(rsp.blockDataLen);
- }
-
- return 0;
-}
-#endif
-
int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
-// void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg));
-// int32_t len = msgLen - sizeof(SSubmitReq2Msg);
-// int32_t vgId = TD_VID(pTq->pVnode);
if (msgType == TDMT_VND_SUBMIT) {
tqProcessSubmitReqForSubscribe(pTq);
- // lock push mgr to avoid potential msg lost
-// taosWLockLatch(&pTq->lock);
-//
-// int32_t numOfRegisteredPush = taosHashGetSize(pTq->pPushMgr);
-// if (numOfRegisteredPush > 0) {
-// tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d",
-// vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush);
-//
-// void* data = taosMemoryMalloc(len);
-// if (data == NULL) {
-// terrno = TSDB_CODE_OUT_OF_MEMORY;
-// tqError("failed to copy data for stream since out of memory, vgId:%d", vgId);
-// taosWUnLockLatch(&pTq->lock);
-// return -1;
-// }
-//
-// memcpy(data, pReq, len);
-//
-// SArray* cachedKey = taosArrayInit(0, sizeof(SItem));
-// void* pIter = NULL;
-//
-// while (1) {
-// pIter = taosHashIterate(pTq->pPushMgr, pIter);
-// if (pIter == NULL) {
-// break;
-// }
-//
-// STqPushEntry* pPushEntry = *(STqPushEntry**)pIter;
-//
-// STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey));
-// if (pHandle == NULL) {
-// tqDebug("vgId:%d, failed to find handle %s in pushing data to consumer, ignore", pTq->pVnode->config.vgId,
-// pPushEntry->subKey);
-// continue;
-// }
-//
-// STqExecHandle* pExec = &pHandle->execHandle;
-// doPushDataForEntry(pIter, pExec, pTq, ver, vgId, data, len, cachedKey);
-// }
-//
-// doRemovePushedEntry(cachedKey, pTq);
-// taosArrayDestroyEx(cachedKey, freeItem);
-// taosMemoryFree(data);
-// }
-//
-// // unlock
-// taosWUnLockLatch(&pTq->lock);
}
- tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, (int)taosHashGetSize(pTq->pStreamMeta->pTasks));
+ int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta);
+ tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, numOfTasks);
// push data for stream processing:
// 1. the vnode has already been restored.
// 2. the vnode should be the leader.
// 3. the stream is not suspended yet.
if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode) && pTq->pVnode->restored) {
- if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) {
+ if (numOfTasks == 0) {
return 0;
}
if (msgType == TDMT_VND_SUBMIT) {
- SPackedData submit = {0};
- tqProcessSubmitReq(pTq, submit);
+ tqStartStreamTasks(pTq);
}
if (msgType == TDMT_VND_DELETE) {
@@ -286,16 +46,16 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
return 0;
}
-
-int32_t tqRegisterPushEntry(STQ* pTq, void* handle, SRpcMsg* pMsg) {
+int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) {
int32_t vgId = TD_VID(pTq->pVnode);
- STqHandle* pHandle = (STqHandle*) handle;
- if(pHandle->msg == NULL){
+ STqHandle* pHandle = (STqHandle*)handle;
+
+ if (pHandle->msg == NULL) {
pHandle->msg = taosMemoryCalloc(1, sizeof(SRpcMsg));
memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg));
pHandle->msg->pCont = rpcMallocCont(pMsg->contLen);
- }else{
- void *tmp = pHandle->msg->pCont;
+ } else {
+ void* tmp = pHandle->msg->pCont;
memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg));
pHandle->msg->pCont = tmp;
}
@@ -303,7 +63,8 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* handle, SRpcMsg* pMsg) {
memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen);
pHandle->msg->contLen = pMsg->contLen;
int32_t ret = taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey), &pHandle, POINTER_BYTES);
- tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen);
+ tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret,
+ pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen);
return 0;
}
@@ -313,6 +74,7 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) {
int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey));
tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId);
+
if(pHandle->msg != NULL) {
tqPushDataRsp(pTq, pHandle);
@@ -320,5 +82,6 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) {
taosMemoryFree(pHandle->msg);
pHandle->msg = NULL;
}
+
return 0;
}
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index ead00dcc35..1fbdb25528 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
+#include "tmsg.h"
#include "tq.h"
bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
@@ -265,9 +266,9 @@ STqReader* tqReaderOpen(SVnode* pVnode) {
pReader->pColIdList = NULL;
pReader->cachedSchemaVer = 0;
pReader->cachedSchemaSuid = 0;
- pReader->pSchema = NULL;
pReader->pSchemaWrapper = NULL;
pReader->tbIdHash = NULL;
+ pReader->pResBlock = createDataBlock();
return pReader;
}
@@ -276,19 +277,19 @@ void tqCloseReader(STqReader* pReader) {
if (pReader->pWalReader) {
walCloseReader(pReader->pWalReader);
}
- // free cached schema
- if (pReader->pSchema) {
- taosMemoryFree(pReader->pSchema);
- }
+
if (pReader->pSchemaWrapper) {
- tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
+ tDeleteSchemaWrapper(pReader->pSchemaWrapper);
}
+
if (pReader->pColIdList) {
taosArrayDestroy(pReader->pColIdList);
}
+
// free hash
+ blockDataDestroy(pReader->pResBlock);
taosHashCleanup(pReader->tbIdHash);
- tDestroySSubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
+ tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
taosMemoryFree(pReader);
}
@@ -322,58 +323,111 @@ int32_t extractSubmitMsgFromWal(SWalReader* pReader, SPackedData* pPackedData) {
return 0;
}
-int32_t tqNextBlock(STqReader* pReader, SSDataBlock* pBlock) {
- while (1) {
- if (pReader->msg2.msgStr == NULL) {
- if (walNextValidMsg(pReader->pWalReader) < 0) {
+// todo ignore the error in wal?
+int32_t tqNextBlockInWal(STqReader* pReader) {
+ SWalReader* pWalReader = pReader->pWalReader;
+
+ while(1) {
+ SArray* pBlockList = pReader->submit.aSubmitTbData;
+ if (pBlockList == NULL || pReader->nextBlk >= taosArrayGetSize(pBlockList)) {
+
+ // try next message in wal file
+ // todo always retry to avoid read failure caused by wal file deletion
+ if (walNextValidMsg(pWalReader) < 0) {
return FETCH_TYPE__NONE;
}
- void* pBody = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
- int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
- int64_t ver = pReader->pWalReader->pHead->head.version;
+ void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
+ int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
+ int64_t ver = pWalReader->pHead->head.version;
- tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver);
- }
+ SDecoder decoder = {0};
+ tDecoderInit(&decoder, pBody, bodyLen);
- while (tqNextBlockImpl(pReader)) {
- memset(pBlock, 0, sizeof(SSDataBlock));
- int32_t code = tqRetrieveDataBlock(pBlock, pReader, NULL);
- if (code != TSDB_CODE_SUCCESS || pBlock->info.rows == 0) {
- continue;
+ {
+ int32_t nSubmitTbData = taosArrayGetSize(pReader->submit.aSubmitTbData);
+ for (int32_t i = 0; i < nSubmitTbData; i++) {
+ SSubmitTbData* pData = taosArrayGet(pReader->submit.aSubmitTbData, i);
+ if (pData->pCreateTbReq != NULL) {
+ taosArrayDestroy(pData->pCreateTbReq->ctb.tagName);
+ taosMemoryFreeClear(pData->pCreateTbReq);
+ }
+ pData->aRowP = taosArrayDestroy(pData->aRowP);
+ }
+ pReader->submit.aSubmitTbData = taosArrayDestroy(pReader->submit.aSubmitTbData);
}
- return FETCH_TYPE__DATA;
+ if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) {
+ tDecoderClear(&decoder);
+ tqError("decode wal file error, msgLen:%d, ver:%"PRId64, bodyLen, ver);
+ return FETCH_TYPE__NONE;
+ }
+
+ tDecoderClear(&decoder);
+ pReader->nextBlk = 0;
}
+
+ size_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData);
+ while (pReader->nextBlk < numOfBlocks) {
+ tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg.msgStr, pReader->msg.msgLen,
+ pReader->msg.ver, pReader->nextBlk);
+
+ SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
+
+ if (pReader->tbIdHash == NULL) {
+ int32_t code = tqRetrieveDataBlock(pReader, NULL);
+ if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) {
+ return FETCH_TYPE__DATA;
+ }
+ }
+
+ void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t));
+ if (ret != NULL) {
+ tqDebug("tq reader return submit block, uid:%"PRId64", ver:%"PRId64, pSubmitTbData->uid, pReader->msg.ver);
+
+ int32_t code = tqRetrieveDataBlock(pReader, NULL);
+ if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) {
+ return FETCH_TYPE__DATA;
+ }
+ } else {
+ pReader->nextBlk += 1;
+ tqDebug("tq reader discard submit block, uid:%"PRId64", continue", pSubmitTbData->uid);
+ }
+ }
+
+ tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
+ pReader->msg.msgStr = NULL;
}
}
int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) {
- pReader->msg2.msgStr = msgStr;
- pReader->msg2.msgLen = msgLen;
- pReader->msg2.ver = ver;
+ pReader->msg.msgStr = msgStr;
+ pReader->msg.msgLen = msgLen;
+ pReader->msg.ver = ver;
tqDebug("tq reader set msg %p %d", msgStr, msgLen);
SDecoder decoder;
- tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen);
- if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) {
+
+ tDecoderInit(&decoder, pReader->msg.msgStr, pReader->msg.msgLen);
+ if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) {
tDecoderClear(&decoder);
tqError("DecodeSSubmitReq2 error, msgLen:%d, ver:%"PRId64, msgLen, ver);
return -1;
}
+
tDecoderClear(&decoder);
return 0;
}
bool tqNextBlockImpl(STqReader* pReader) {
- if (pReader->msg2.msgStr == NULL) {
+ if (pReader->msg.msgStr == NULL) {
return false;
}
int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData);
while (pReader->nextBlk < blockSz) {
- tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen,
- pReader->msg2.ver, pReader->nextBlk);
+ tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg.msgStr, pReader->msg.msgLen,
+ pReader->msg.ver, pReader->nextBlk);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if (pReader->tbIdHash == NULL) {
@@ -382,7 +436,7 @@ bool tqNextBlockImpl(STqReader* pReader) {
void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t));
if (ret != NULL) {
- tqDebug("tq reader block found, ver:%"PRId64", uid:%"PRId64, pReader->msg2.ver, pSubmitTbData->uid);
+ tqDebug("tq reader block found, ver:%"PRId64", uid:%"PRId64, pReader->msg.ver, pSubmitTbData->uid);
return true;
} else {
tqDebug("tq reader discard submit block, uid:%"PRId64", continue", pSubmitTbData->uid);
@@ -391,15 +445,15 @@ bool tqNextBlockImpl(STqReader* pReader) {
pReader->nextBlk++;
}
- tDestroySSubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
+ tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
pReader->nextBlk = 0;
- pReader->msg2.msgStr = NULL;
+ pReader->msg.msgStr = NULL;
return false;
}
bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) {
- if (pReader->msg2.msgStr == NULL) return false;
+ if (pReader->msg.msgStr == NULL) return false;
int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData);
while (pReader->nextBlk < blockSz) {
@@ -413,9 +467,9 @@ bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) {
pReader->nextBlk++;
}
- tDestroySSubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
+ tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
pReader->nextBlk = 0;
- pReader->msg2.msgStr = NULL;
+ pReader->msg.msgStr = NULL;
return false;
}
@@ -449,212 +503,219 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap
return 0;
}
-int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) {
- tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg2.msgStr, pReader->nextBlk);
- SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
- pReader->nextBlk++;
+int32_t tqRetrieveDataBlock(STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) {
+ tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk);
+ SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++);
if (pSubmitTbDataRet) {
*pSubmitTbDataRet = pSubmitTbData;
}
+ SSDataBlock* pBlock = pReader->pResBlock;
+ blockDataCleanup(pBlock);
+
int32_t sversion = pSubmitTbData->sver;
int64_t suid = pSubmitTbData->suid;
int64_t uid = pSubmitTbData->uid;
pReader->lastBlkUid = uid;
pBlock->info.id.uid = uid;
- pBlock->info.version = pReader->msg2.ver;
+ pBlock->info.version = pReader->msg.ver;
- if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || pReader->cachedSchemaSuid != suid) {
- taosMemoryFree(pReader->pSchema);
- pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1);
- if (pReader->pSchema == NULL) {
- tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64
- "), version %d, possibly dropped table",
- pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion);
- pReader->cachedSchemaSuid = 0;
- terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
- return -1;
- }
+ if ((suid != 0 && pReader->cachedSchemaSuid != suid) || (suid == 0 && pReader->cachedSchemaUid != uid) || (pReader->cachedSchemaVer != sversion)) {
+ tDeleteSchemaWrapper(pReader->pSchemaWrapper);
- tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
if (pReader->pSchemaWrapper == NULL) {
- tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
- pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer);
+ tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64 "version %d, possibly dropped table",
+ pReader->pWalReader->pWal->cfg.vgId, suid, uid, pReader->cachedSchemaVer);
pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1;
}
- STSchema* pTschema = pReader->pSchema;
+ pReader->cachedSchemaUid = uid;
+ pReader->cachedSchemaSuid = suid;
+ pReader->cachedSchemaVer = sversion;
+
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
+ if (blockDataGetNumOfCols(pBlock) > 0) {
+ blockDataDestroy(pReader->pResBlock);
+ pReader->pResBlock = createDataBlock();
+ pBlock = pReader->pResBlock;
- int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList);
+ pBlock->info.id.uid = uid;
+ pBlock->info.version = pReader->msg.ver;
+ }
- if (colNumNeed == 0) {
- int32_t colMeta = 0;
- while (colMeta < pSchemaWrapper->nCols) {
- SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
+ int32_t numOfCols = taosArrayGetSize(pReader->pColIdList);
+ if (numOfCols == 0) { // all columns are required
+ for (int32_t i = 0; i < pSchemaWrapper->nCols; ++i) {
+ SSchema* pColSchema = &pSchemaWrapper->pSchema[i];
SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
- int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
+
+ int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS) {
- goto FAIL;
+ blockDataFreeRes(pBlock);
+ return -1;
}
- colMeta++;
}
} else {
- if (colNumNeed > pSchemaWrapper->nCols) {
- colNumNeed = pSchemaWrapper->nCols;
+ if (numOfCols > pSchemaWrapper->nCols) {
+ numOfCols = pSchemaWrapper->nCols;
}
- int32_t colMeta = 0;
- int32_t colNeed = 0;
- while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
- SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
+ int32_t i = 0;
+ int32_t j = 0;
+ while (i < pSchemaWrapper->nCols && j < numOfCols) {
+ SSchema* pColSchema = &pSchemaWrapper->pSchema[i];
col_id_t colIdSchema = pColSchema->colId;
- col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed);
+
+ col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, j);
if (colIdSchema < colIdNeed) {
- colMeta++;
+ i++;
} else if (colIdSchema > colIdNeed) {
- colNeed++;
+ j++;
} else {
SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId);
int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS) {
- goto FAIL;
+ return -1;
}
- colMeta++;
- colNeed++;
+ i++;
+ j++;
}
}
}
+ }
- int32_t numOfRows = 0;
+ int32_t numOfRows = 0;
+ if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
+ SColData* pCol = taosArrayGet(pSubmitTbData->aCol, 0);
+ numOfRows = pCol->nVal;
+ } else {
+ numOfRows = taosArrayGetSize(pSubmitTbData->aRowP);
+ }
- if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
- SArray* pCols = pSubmitTbData->aCol;
- SColData* pCol = taosArrayGet(pCols, 0);
- numOfRows = pCol->nVal;
- } else {
- SArray* pRows = pSubmitTbData->aRowP;
- numOfRows = taosArrayGetSize(pRows);
+ if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ return -1;
+ }
+
+ pBlock->info.rows = numOfRows;
+
+ int32_t colActual = blockDataGetNumOfCols(pBlock);
+
+ // convert and scan one block
+ if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
+ SArray* pCols = pSubmitTbData->aCol;
+ int32_t numOfCols = taosArrayGetSize(pCols);
+ int32_t targetIdx = 0;
+ int32_t sourceIdx = 0;
+ while (targetIdx < colActual) {
+ if (sourceIdx >= numOfCols) {
+ tqError("tqRetrieveDataBlock sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols);
+ return -1;
+ }
+
+ SColData* pCol = taosArrayGet(pCols, sourceIdx);
+ SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
+ SColVal colVal;
+
+ if (pCol->nVal != numOfRows) {
+ tqError("tqRetrieveDataBlock pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows);
+ return -1;
+ }
+
+ if (pCol->cid < pColData->info.colId) {
+ sourceIdx++;
+ } else if (pCol->cid == pColData->info.colId) {
+ for (int32_t i = 0; i < pCol->nVal; i++) {
+ tColDataGetValue(pCol, i, &colVal);
+ if (IS_STR_DATA_TYPE(colVal.type)) {
+ if (colVal.value.pData != NULL) {
+ char val[65535 + 2] = {0};
+ memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData);
+ varDataSetLen(val, colVal.value.nData);
+ if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
+ return -1;
+ }
+ } else {
+ colDataSetNULL(pColData, i);
+ }
+ } else {
+ if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
+ return -1;
+ }
+ }
+ }
+ sourceIdx++;
+ targetIdx++;
+ } else {
+ for (int32_t i = 0; i < pCol->nVal; i++) {
+ colDataSetNULL(pColData, i);
+ }
+
+ targetIdx++;
+ }
}
+ } else {
+ SArray* pRows = pSubmitTbData->aRowP;
+ SSchemaWrapper* pWrapper = pReader->pSchemaWrapper;
+ STSchema* pTSchema = tBuildTSchema(pWrapper->pSchema, pWrapper->nCols, pWrapper->version);
- if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) {
- terrno = TSDB_CODE_OUT_OF_MEMORY;
- goto FAIL;
- }
- pBlock->info.rows = numOfRows;
-
- int32_t colActual = blockDataGetNumOfCols(pBlock);
-
- // convert and scan one block
- if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
- SArray* pCols = pSubmitTbData->aCol;
- int32_t numOfCols = taosArrayGetSize(pCols);
- int32_t targetIdx = 0;
+ for (int32_t i = 0; i < numOfRows; i++) {
+ SRow* pRow = taosArrayGetP(pRows, i);
int32_t sourceIdx = 0;
- while (targetIdx < colActual) {
- if(sourceIdx >= numOfCols){
- tqError("tqRetrieveDataBlock sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols);
- goto FAIL;
- }
- SColData* pCol = taosArrayGet(pCols, sourceIdx);
- SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
- SColVal colVal;
- if(pCol->nVal != numOfRows){
- tqError("tqRetrieveDataBlock pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows);
- goto FAIL;
- }
+ for (int32_t j = 0; j < colActual; j++) {
+ SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j);
+ while (1) {
+ SColVal colVal;
+ tqDebug("start to extract column id:%d, index:%d", pColData->info.colId, sourceIdx);
- if (pCol->cid < pColData->info.colId) {
- sourceIdx++;
- } else if (pCol->cid == pColData->info.colId) {
- for (int32_t i = 0; i < pCol->nVal; i++) {
- tColDataGetValue(pCol, i, &colVal);
+ tRowGet(pRow, pTSchema, sourceIdx, &colVal);
+ if (colVal.cid < pColData->info.colId) {
+ tqDebug("colIndex:%d column id:%d in row, ignore, the required colId:%d, total cols in schema:%d",
+ sourceIdx, colVal.cid, pColData->info.colId, pTSchema->numOfCols);
+ sourceIdx++;
+ continue;
+ } else if (colVal.cid == pColData->info.colId) {
if (IS_STR_DATA_TYPE(colVal.type)) {
if (colVal.value.pData != NULL) {
char val[65535 + 2] = {0};
memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData);
varDataSetLen(val, colVal.value.nData);
if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
- goto FAIL;
+ return -1;
}
} else {
colDataSetNULL(pColData, i);
}
} else {
if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
- goto FAIL;
+ return -1;
}
}
- }
- sourceIdx++;
- targetIdx++;
- } else {
- for (int32_t i = 0; i < pCol->nVal; i++) {
+
+ sourceIdx++;
+ break;
+ } else {
colDataSetNULL(pColData, i);
- }
- targetIdx++;
- }
- }
- } else {
- SArray* pRows = pSubmitTbData->aRowP;
-
- for (int32_t i = 0; i < numOfRows; i++) {
- SRow* pRow = taosArrayGetP(pRows, i);
- int32_t sourceIdx = 0;
-
- for (int32_t j = 0; j < colActual; j++) {
- SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j);
- while (1) {
- SColVal colVal;
- tRowGet(pRow, pTschema, sourceIdx, &colVal);
- if (colVal.cid < pColData->info.colId) {
- sourceIdx++;
- continue;
- } else if (colVal.cid == pColData->info.colId) {
- if (IS_STR_DATA_TYPE(colVal.type)) {
- if (colVal.value.pData != NULL) {
- char val[65535 + 2] = {0};
- memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData);
- varDataSetLen(val, colVal.value.nData);
- if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
- goto FAIL;
- }
- } else {
- colDataSetNULL(pColData, i);
- }
- } else {
- if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) {
- goto FAIL;
- }
- }
-
- sourceIdx++;
- break;
- } else {
- colDataSetNULL(pColData, i);
- break;
- }
+ break;
}
}
}
}
+
+ taosMemoryFreeClear(pTSchema);
}
return 0;
-
-FAIL:
- blockDataFreeRes(pBlock);
- return -1;
}
int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) {
- tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk);
+ tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
pReader->nextBlk++;
@@ -665,18 +726,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
int64_t uid = pSubmitTbData->uid;
pReader->lastBlkUid = uid;
- taosMemoryFree(pReader->pSchema);
- pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1);
- if (pReader->pSchema == NULL) {
- tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64
- "), version %d, possibly dropped table",
- pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion);
- pReader->cachedSchemaSuid = 0;
- terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
- return -1;
- }
-
- tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
+ tDeleteSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1);
if (pReader->pSchemaWrapper == NULL) {
tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
@@ -686,7 +736,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
return -1;
}
- STSchema* pTschema = pReader->pSchema;
SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper;
int32_t numOfRows = 0;
@@ -743,18 +792,18 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
if (tqMaskBlock(pSW, &block, pSchemaWrapper, assigned) < 0) {
blockDataFreeRes(&block);
- tDeleteSSchemaWrapper(pSW);
+ tDeleteSchemaWrapper(pSW);
goto FAIL;
}
tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId,
(int32_t)taosArrayGetSize(block.pDataBlock));
block.info.id.uid = uid;
- block.info.version = pReader->msg2.ver;
+ block.info.version = pReader->msg.ver;
if (blockDataEnsureCapacity(&block, numOfRows - curRow) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
blockDataFreeRes(&block);
- tDeleteSSchemaWrapper(pSW);
+ tDeleteSchemaWrapper(pSW);
goto FAIL;
}
taosArrayPush(blocks, &block);
@@ -803,14 +852,17 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
curRow++;
}
} else {
+ SSchemaWrapper* pWrapper = pReader->pSchemaWrapper;
+ STSchema* pTSchema = tBuildTSchema(pWrapper->pSchema, pWrapper->nCols, pWrapper->version);
SArray* pRows = pSubmitTbData->aRowP;
+
for (int32_t i = 0; i < numOfRows; i++) {
SRow* pRow = taosArrayGetP(pRows, i);
bool buildNew = false;
- for (int32_t j = 0; j < pTschema->numOfCols; j++) {
+ for (int32_t j = 0; j < pTSchema->numOfCols; j++) {
SColVal colVal;
- tRowGet(pRow, pTschema, j, &colVal);
+ tRowGet(pRow, pTSchema, j, &colVal);
if (curRow == 0) {
assigned[j] = !COL_VAL_IS_NONE(&colVal);
buildNew = true;
@@ -839,18 +891,18 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
if (tqMaskBlock(pSW, &block, pSchemaWrapper, assigned) < 0) {
blockDataFreeRes(&block);
- tDeleteSSchemaWrapper(pSW);
+ tDeleteSchemaWrapper(pSW);
goto FAIL;
}
tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId,
(int32_t)taosArrayGetSize(block.pDataBlock));
block.info.id.uid = uid;
- block.info.version = pReader->msg2.ver;
+ block.info.version = pReader->msg.ver;
if (blockDataEnsureCapacity(&block, numOfRows - curRow) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
blockDataFreeRes(&block);
- tDeleteSSchemaWrapper(pSW);
+ tDeleteSchemaWrapper(pSW);
goto FAIL;
}
taosArrayPush(blocks, &block);
@@ -868,7 +920,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
while (targetIdx < colActual) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
SColVal colVal;
- tRowGet(pRow, pTschema, sourceIdx, &colVal);
+ tRowGet(pRow, pTSchema, sourceIdx, &colVal);
if (colVal.cid < pColData->info.colId) {
sourceIdx++;
@@ -895,6 +947,8 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
}
curRow++;
}
+
+ taosMemoryFreeClear(pTSchema);
}
SSDataBlock* pLastBlock = taosArrayGetLast(blocks);
@@ -1039,6 +1093,5 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
}
taosWUnLockLatch(&pTq->pStreamMeta->lock);
-
return 0;
}
diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c
index 58cb7b9e63..0bb33b1215 100644
--- a/source/dnode/vnode/src/tq/tqRestore.c
+++ b/source/dnode/vnode/src/tq/tqRestore.c
@@ -36,6 +36,7 @@ int32_t tqStreamTasksScanWal(STQ* pTq) {
if (shouldIdle) {
taosWLockLatch(&pMeta->lock);
+
pMeta->walScanCounter -= 1;
times = pMeta->walScanCounter;
@@ -56,42 +57,28 @@ int32_t tqStreamTasksScanWal(STQ* pTq) {
return 0;
}
-static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) {
- SArray* pTaskIdList = taosArrayInit(numOfTasks, sizeof(int32_t));
- void* pIter = NULL;
-
- taosWLockLatch(&pStreamMeta->lock);
- while(1) {
- pIter = taosHashIterate(pStreamMeta->pTasks, pIter);
- if (pIter == NULL) {
- break;
- }
-
- SStreamTask* pTask = *(SStreamTask**)pIter;
- taosArrayPush(pTaskIdList, &pTask->id.taskId);
- }
-
- taosWUnLockLatch(&pStreamMeta->lock);
- return pTaskIdList;
-}
-
int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
*pScanIdle = true;
bool noNewDataInWal = true;
int32_t vgId = pStreamMeta->vgId;
- int32_t numOfTasks = taosHashGetSize(pStreamMeta->pTasks);
+ int32_t numOfTasks = taosArrayGetSize(pStreamMeta->pTaskList);
if (numOfTasks == 0) {
return TSDB_CODE_SUCCESS;
}
+ SArray* pTaskList = NULL;
+ taosWLockLatch(&pStreamMeta->lock);
+ pTaskList = taosArrayDup(pStreamMeta->pTaskList, NULL);
+ taosWUnLockLatch(&pStreamMeta->lock);
+
tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks);
- SArray* pTaskIdList = extractTaskIdList(pStreamMeta, numOfTasks);
// update the new task number
- numOfTasks = taosArrayGetSize(pTaskIdList);
+ numOfTasks = taosArrayGetSize(pTaskList);
+
for (int32_t i = 0; i < numOfTasks; ++i) {
- int32_t* pTaskId = taosArrayGet(pTaskIdList, i);
+ int32_t* pTaskId = taosArrayGet(pTaskList, i);
SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId);
if (pTask == NULL) {
continue;
@@ -122,6 +109,15 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
// seek the stored version and extract data from WAL
int32_t code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer);
if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit
+ SWal *pWal = pTask->exec.pWalReader->pWal;
+ if (pTask->chkInfo.currentVer < pWal->vers.firstVer ) {
+ pTask->chkInfo.currentVer = pWal->vers.firstVer;
+ code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer);
+ if (code != TSDB_CODE_SUCCESS) {
+ streamMetaReleaseTask(pStreamMeta, pTask);
+ continue;
+ }
+ }
streamMetaReleaseTask(pStreamMeta, pTask);
continue;
}
@@ -165,7 +161,7 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
*pScanIdle = true;
}
- taosArrayDestroy(pTaskIdList);
+ taosArrayDestroy(pTaskList);
return 0;
}
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index 8e243a8bd1..800bcc8b71 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -66,9 +66,10 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in
int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
const int32_t MAX_ROWS_TO_RETURN = 4096;
- int32_t vgId = TD_VID(pTq->pVnode);
- int32_t code = 0;
- int32_t totalRows = 0;
+
+ int32_t vgId = TD_VID(pTq->pVnode);
+ int32_t code = 0;
+ int32_t totalRows = 0;
const STqExecHandle* pExec = &pHandle->execHandle;
qTaskInfo_t task = pExec->task;
@@ -215,7 +216,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
int64_t uid = pExec->pTqReader->lastBlkUid;
if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes);
- taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper);
+ taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper);
pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
pSchemas = taosArrayInit(0, sizeof(void*));
continue;
@@ -274,7 +275,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
int64_t uid = pExec->pTqReader->lastBlkUid;
if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes);
- taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper);
+ taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper);
pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
pSchemas = taosArrayInit(0, sizeof(void*));
continue;
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index c2e6946b04..33d1e08c9c 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -672,7 +672,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
}
SRow* pRow = NULL;
if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) {
- tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
goto _end;
}
ASSERT(pRow);
@@ -681,7 +681,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
SSubmitReq2 submitReq = {0};
if (!(submitReq.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) {
- tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
goto _end;
}
@@ -690,28 +690,28 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void*
// encode
int32_t len;
int32_t code;
- tEncodeSize(tEncodeSSubmitReq2, &submitReq, len, code);
+ tEncodeSize(tEncodeSubmitReq, &submitReq, len, code);
SEncoder encoder;
len += sizeof(SSubmitReq2Msg);
pBuf = rpcMallocCont(len);
if (NULL == pBuf) {
- tDestroySSubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
goto _end;
}
((SSubmitReq2Msg*)pBuf)->header.vgId = TD_VID(pVnode);
((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len);
((SSubmitReq2Msg*)pBuf)->version = htobe64(1);
tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg));
- if (tEncodeSSubmitReq2(&encoder, &submitReq) < 0) {
+ if (tEncodeSubmitReq(&encoder, &submitReq) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tqError("failed to encode submit req since %s", terrstr());
tEncoderClear(&encoder);
rpcFreeCont(pBuf);
- tDestroySSubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
continue;
}
tEncoderClear(&encoder);
- tDestroySSubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
SRpcMsg msg = {
.msgType = TDMT_VND_SUBMIT,
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index d186c63871..a66d63a910 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -165,22 +165,29 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
SRpcMsg* pMsg, STqOffsetVal* pOffset) {
uint64_t consumerId = pRequest->consumerId;
int32_t vgId = TD_VID(pTq->pVnode);
+ int code = 0;
SMqDataRsp dataRsp = {0};
tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType);
+ qTaskInfo_t task = pHandle->execHandle.task;
+ if(qTaskIsExecuting(task)){
+ code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP);
+ tDeleteSMqDataRsp(&dataRsp);
+ return code;
+ }
qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
- int code = tqScanData(pTq, pHandle, &dataRsp, pOffset);
+ code = tqScanData(pTq, pHandle, &dataRsp, pOffset);
if(code != 0) {
goto end;
}
-// till now, all data has been transferred to consumer, new data needs to push client once arrived.
+ // till now, all data has been transferred to consumer, new data needs to push client once arrived.
if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) {
// lock
taosWLockLatch(&pTq->lock);
- code = tqRegisterPushEntry(pTq, pHandle, pMsg);
+ code = tqRegisterPushHandle(pTq, pHandle, pMsg);
taosWUnLockLatch(&pTq->lock);
tDeleteSMqDataRsp(&dataRsp);
return code;
@@ -246,6 +253,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
if (offset->type == TMQ_OFFSET__LOG) {
+ verifyOffset(pHandle->pWalReader, offset);
int64_t fetchVer = offset->version + 1;
pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
if (pCkHead == NULL) {
@@ -361,11 +369,10 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ
// this is a normal subscribe requirement
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
+ } else { // todo handle the case where re-balance occurs.
+ // for taosx
+ return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
}
-
- // todo handle the case where re-balance occurs.
- // for taosx
- return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset);
}
int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 1ef86f5b30..2fc2b8cdd6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -21,10 +21,9 @@
#define getCurrentKeyInLastBlock(_r) ((_r)->currentKey)
typedef enum {
- READER_STATUS_SUSPEND = 0x1,
- READER_STATUS_SHOULD_STOP = 0x2,
- READER_STATUS_NORMAL = 0x3,
-} EReaderExecStatus;
+ READER_STATUS_SUSPEND = 0x1,
+ READER_STATUS_NORMAL = 0x2,
+} EReaderStatus;
typedef enum {
EXTERNAL_ROWS_PREV = 0x1,
@@ -184,6 +183,7 @@ typedef struct STsdbReaderAttr {
STimeWindow window;
bool freeBlock;
SVersionRange verRange;
+ int16_t order;
} STsdbReaderAttr;
typedef struct SResultBlockInfo {
@@ -196,7 +196,8 @@ struct STsdbReader {
STsdb* pTsdb;
SVersionRange verRange;
TdThreadMutex readerMutex;
- EReaderExecStatus flag;
+ EReaderStatus flag;
+ int32_t code;
uint64_t suid;
int16_t order;
EReadMode readMode;
@@ -2995,9 +2996,9 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr
while (1) {
// only check here, since the iterate data in memory is very fast.
- if (pReader->flag == READER_STATUS_SHOULD_STOP) {
- tsdbWarn("tsdb reader is stopped ASAP, %s", pReader->idStr);
- return TSDB_CODE_SUCCESS;
+ if (pReader->code != TSDB_CODE_SUCCESS) {
+ tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr);
+ return pReader->code;
}
bool hasNext = false;
@@ -3093,9 +3094,9 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
while (1) {
- if (pReader->flag == READER_STATUS_SHOULD_STOP) {
- tsdbWarn("tsdb reader is stopped ASAP, %s", pReader->idStr);
- return TSDB_CODE_SUCCESS;
+ if (pReader->code != TSDB_CODE_SUCCESS) {
+ tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr);
+ return pReader->code;
}
// load the last data block of current table
@@ -3246,7 +3247,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
}
}
- return code;
+ return (pReader->code != TSDB_CODE_SUCCESS)? pReader->code:code;
}
static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) {
@@ -3395,9 +3396,9 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) {
STableUidList* pUidList = &pStatus->uidList;
while (1) {
- if (pReader->flag == READER_STATUS_SHOULD_STOP) {
- tsdbWarn("tsdb reader is stopped ASAP, %s", pReader->idStr);
- return TSDB_CODE_SUCCESS;
+ if (pReader->code != TSDB_CODE_SUCCESS) {
+ tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr);
+ return pReader->code;
}
STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter;
@@ -3493,7 +3494,7 @@ static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) {
terrno = 0;
code = doLoadLastBlockSequentially(pReader);
- if (code != TSDB_CODE_SUCCESS || pReader->flag == READER_STATUS_SHOULD_STOP) {
+ if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return TSDB_READ_RETURN;
}
@@ -3507,8 +3508,7 @@ static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) {
code = initForFirstBlockInFile(pReader, pBlockIter);
// error happens or all the data files are completely checked
- if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false) ||
- pReader->flag == READER_STATUS_SHOULD_STOP) {
+ if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) {
terrno = code;
return TSDB_READ_RETURN;
}
@@ -3536,13 +3536,9 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
}
code = doBuildDataBlock(pReader);
- if (code != TSDB_CODE_SUCCESS || pReader->flag == READER_STATUS_SHOULD_STOP) {
+ if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) {
return code;
}
-
- if (pResBlock->info.rows > 0) {
- return TSDB_CODE_SUCCESS;
- }
}
while (1) {
@@ -3581,13 +3577,9 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
code = doBuildDataBlock(pReader);
}
- if (code != TSDB_CODE_SUCCESS || pReader->flag == READER_STATUS_SHOULD_STOP) {
+ if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) {
return code;
}
-
- if (pResBlock->info.rows > 0) {
- return TSDB_CODE_SUCCESS;
- }
}
}
@@ -4849,8 +4841,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) {
*hasNext = false;
- if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT) {
- return code;
+ if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT || pReader->code != TSDB_CODE_SUCCESS) {
+ return (pReader->code != TSDB_CODE_SUCCESS)? pReader->code:code;
}
SReaderStatus* pStatus = &pReader->status;
@@ -4859,7 +4851,11 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) {
qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code);
if (pReader->flag == READER_STATUS_SUSPEND) {
- tsdbReaderResume(pReader);
+ code = tsdbReaderResume(pReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ tsdbReleaseReader(pReader);
+ return code;
+ }
}
if (pReader->innerReader[0] != NULL && pReader->step == 0) {
@@ -5132,11 +5128,17 @@ SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
}
int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
+ int32_t code = TSDB_CODE_SUCCESS;
+
qTrace("tsdb/reader-reset: %p, take read mutex", pReader);
tsdbAcquireReader(pReader);
if (pReader->flag == READER_STATUS_SUSPEND) {
- tsdbReaderResume(pReader);
+ code = tsdbReaderResume(pReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ tsdbReleaseReader(pReader);
+ return code;
+ }
}
if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) {
@@ -5171,8 +5173,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
int64_t ts = asc ? pReader->window.skey - 1 : pReader->window.ekey + 1;
resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step);
- int32_t code = 0;
-
// no data in files, let's try buffer in memory
if (pStatus->fileIter.numOfFiles == 0) {
pStatus->loadFromFile = false;
@@ -5217,7 +5217,11 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
// find the start data block in file
tsdbAcquireReader(pReader);
if (pReader->flag == READER_STATUS_SUSPEND) {
- tsdbReaderResume(pReader);
+ code = tsdbReaderResume(pReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ tsdbReleaseReader(pReader);
+ return code;
+ }
}
SReaderStatus* pStatus = &pReader->status;
@@ -5285,12 +5289,17 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
}
int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) {
+ int32_t code = TSDB_CODE_SUCCESS;
int64_t rows = 0;
SReaderStatus* pStatus = &pReader->status;
tsdbAcquireReader(pReader);
if (pReader->flag == READER_STATUS_SUSPEND) {
- tsdbReaderResume(pReader);
+ code = tsdbReaderResume(pReader);
+ if (code != TSDB_CODE_SUCCESS) {
+ tsdbReleaseReader(pReader);
+ return code;
+ }
}
int32_t iter = 0;
@@ -5456,4 +5465,4 @@ void tsdbReaderSetId(STsdbReader* pReader, const char* idstr) {
pReader->idStr = taosStrdup(idstr);
}
-void tsdbReaderSetCloseFlag(STsdbReader* pReader) { pReader->flag = READER_STATUS_SHOULD_STOP; }
+void tsdbReaderSetCloseFlag(STsdbReader* pReader) { pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED; }
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 251efeab3d..c608403456 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -448,7 +448,6 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
walApplyVer(pVnode->pWal, version);
if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
-// /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/
vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));
return -1;
}
@@ -1007,7 +1006,7 @@ static int32_t vnodeResetTableCxt(SMeta *pMeta, SSubmitReqConvertCxt *pCxt) {
}
tdSTSRowIterInit(&pCxt->rowIter, pCxt->pTbSchema);
- tDestroySSubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE);
if (NULL == pCxt->pTbData) {
pCxt->pTbData = taosMemoryCalloc(1, sizeof(SSubmitTbData));
if (NULL == pCxt->pTbData) {
@@ -1039,7 +1038,7 @@ static int32_t vnodeResetTableCxt(SMeta *pMeta, SSubmitReqConvertCxt *pCxt) {
static void vnodeDestroySubmitReqConvertCxt(SSubmitReqConvertCxt *pCxt) {
taosMemoryFreeClear(pCxt->pTbSchema);
- tDestroySSubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE);
taosMemoryFreeClear(pCxt->pTbData);
taosArrayDestroy(pCxt->pColValues);
}
@@ -1149,7 +1148,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) {
int32_t code = TSDB_CODE_SUCCESS;
char *pMsg = NULL;
uint32_t msglen = 0;
- tEncodeSize(tEncodeSSubmitReq2, pSubmitReq, msglen, code);
+ tEncodeSize(tEncodeSubmitReq, pSubmitReq, msglen, code);
if (TSDB_CODE_SUCCESS == code) {
pMsg = taosMemoryMalloc(msglen);
if (NULL == pMsg) {
@@ -1159,7 +1158,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) {
if (TSDB_CODE_SUCCESS == code) {
SEncoder encoder;
tEncoderInit(&encoder, pMsg, msglen);
- code = tEncodeSSubmitReq2(&encoder, pSubmitReq);
+ code = tEncodeSubmitReq(&encoder, pSubmitReq);
tEncoderClear(&encoder);
}
if (TSDB_CODE_SUCCESS == code) {
@@ -1199,7 +1198,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
len -= sizeof(SSubmitReq2Msg);
SDecoder dc = {0};
tDecoderInit(&dc, pReq, len);
- if (tDecodeSSubmitReq2(&dc, pSubmitReq) < 0) {
+ if (tDecodeSubmitReq(&dc, pSubmitReq) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
@@ -1388,7 +1387,7 @@ _exit:
// clear
taosArrayDestroy(newTbUids);
- tDestroySSubmitReq(pSubmitReq, 0 == pMsg->version ? TSDB_MSG_FLG_CMPT : TSDB_MSG_FLG_DECODE);
+ tDestroySubmitReq(pSubmitReq, 0 == pMsg->version ? TSDB_MSG_FLG_CMPT : TSDB_MSG_FLG_DECODE);
tDestroySSubmitRsp2(pSubmitRsp, TSDB_MSG_FLG_ENCODE);
if (code) terrno = code;
diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h
index 8852265da0..37c93fef5c 100644
--- a/source/libs/executor/inc/querytask.h
+++ b/source/libs/executor/inc/querytask.h
@@ -59,7 +59,7 @@ typedef struct {
STqOffsetVal currentOffset; // for tmq
SMqMetaRsp metaRsp; // for tmq fetching meta
int64_t snapshotVer;
- SPackedData submit; // todo remove it
+// SPackedData submit; // todo remove it
SSchemaWrapper* schema;
char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor
int8_t recoverStep;
diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c
index 33eccf4759..d31ac0bc51 100644
--- a/source/libs/executor/src/dataInserter.c
+++ b/source/libs/executor/src/dataInserter.c
@@ -126,7 +126,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int
int32_t code = TSDB_CODE_SUCCESS;
int32_t len = 0;
void* pBuf = NULL;
- tEncodeSize(tEncodeSSubmitReq2, pReq, len, code);
+ tEncodeSize(tEncodeSubmitReq, pReq, len, code);
if (TSDB_CODE_SUCCESS == code) {
SEncoder encoder;
len += sizeof(SSubmitReq2Msg);
@@ -138,7 +138,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int
((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len);
((SSubmitReq2Msg*)pBuf)->version = htobe64(1);
tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg));
- code = tEncodeSSubmitReq2(&encoder, pReq);
+ code = tEncodeSubmitReq(&encoder, pReq);
tEncoderClear(&encoder);
}
@@ -281,7 +281,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
SRow* pRow = NULL;
if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) {
- tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
goto _end;
}
taosArrayPush(tbData.aRowP, &pRow);
@@ -301,7 +301,7 @@ _end:
if (terrno != 0) {
*ppReq = NULL;
if (pReq) {
- tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq);
}
return terrno;
@@ -326,7 +326,7 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32
code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid);
if (code) {
if (pReq) {
- tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq);
}
@@ -335,7 +335,7 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32
}
code = submitReqToMsg(vgId, pReq, pMsg, msgLen);
- tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq);
return code;
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 2d991a14f5..8bbbd3524d 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1052,25 +1052,20 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
return TSDB_CODE_SUCCESS;
}
-int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) {
- SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
- if ((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)) {
- qError("qStreamSetScanMemData err:%d,%p", pTaskInfo->execModel, pTaskInfo->streamInfo.submit.msgStr);
- terrno = TSDB_CODE_PAR_INTERNAL_ERROR;
- return -1;
- }
- qDebug("set the submit block for future scan");
-
- pTaskInfo->streamInfo.submit = submit;
- return 0;
-}
-
void qStreamSetOpen(qTaskInfo_t tinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SOperatorInfo* pOperator = pTaskInfo->pRoot;
pOperator->status = OP_NOT_OPENED;
}
+void verifyOffset(void *pWalReader, STqOffsetVal* pOffset){
+ // if offset version is small than first version , let's seek to first version
+ int64_t firstVer = walGetFirstVer(((SWalReader*)pWalReader)->pWal);
+ if (pOffset->version + 1 < firstVer){
+ pOffset->version = firstVer - 1;
+ }
+}
+
int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
SOperatorInfo* pOperator = pTaskInfo->pRoot;
@@ -1086,21 +1081,18 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
if (pOperator == NULL) {
return -1;
}
+
SStreamScanInfo* pInfo = pOperator->info;
STableScanInfo* pScanInfo = pInfo->pTableScanOp->info;
STableScanBase* pScanBaseInfo = &pScanInfo->base;
STableListInfo* pTableListInfo = pScanBaseInfo->pTableListInfo;
if (pOffset->type == TMQ_OFFSET__LOG) {
+ // todo refactor: move away
tsdbReaderClose(pScanBaseInfo->dataReader);
pScanBaseInfo->dataReader = NULL;
- // let's seek to the next version in wal file
- int64_t firstVer = walGetFirstVer(pInfo->tqReader->pWalReader->pWal);
- if (pOffset->version + 1 < firstVer){
- pOffset->version = firstVer - 1;
- }
-
+ verifyOffset(pInfo->tqReader->pWalReader, pOffset);
if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) {
qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
return -1;
@@ -1221,7 +1213,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
- tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
+ tDeleteSchemaWrapper(pTaskInfo->streamInfo.schema);
pTaskInfo->streamInfo.schema = mtInfo.schema;
qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64 " %s", mtInfo.uid, pOffset->ts, id);
diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c
index f525f6728c..62ab2d9df2 100644
--- a/source/libs/executor/src/executorInt.c
+++ b/source/libs/executor/src/executorInt.c
@@ -82,7 +82,7 @@ static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SC
static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol);
static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
- SGroupResInfo* pGroupResInfo);
+ SGroupResInfo* pGroupResInfo, int32_t threshold);
SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize) {
SFilePage* pData = NULL;
@@ -776,7 +776,7 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos
}
int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
- SGroupResInfo* pGroupResInfo) {
+ SGroupResInfo* pGroupResInfo, int32_t threshold) {
SExprInfo* pExprInfo = pSup->pExprInfo;
int32_t numOfExprs = pSup->numOfExprs;
int32_t* rowEntryOffset = pSup->rowEntryInfoOffset;
@@ -825,6 +825,9 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
releaseBufPage(pBuf, page);
pBlock->info.rows += pRow->numOfRows;
+ if (pBlock->info.rows >= threshold) {
+ break;
+ }
}
qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
@@ -850,7 +853,7 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr
// clear the existed group id
pBlock->info.id.groupId = 0;
ASSERT(!pbInfo->mergeResultBlock);
- doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo);
+ doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold);
void* tbname = NULL;
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
@@ -877,10 +880,10 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
// clear the existed group id
pBlock->info.id.groupId = 0;
if (!pbInfo->mergeResultBlock) {
- doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo);
+ doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold);
} else {
while (hasRemainResults(pGroupResInfo)) {
- doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo);
+ doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo, pOperator->resultInfo.threshold);
if (pBlock->info.rows >= pOperator->resultInfo.threshold) {
break;
}
diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c
index a4d8327b6a..7716b5976b 100644
--- a/source/libs/executor/src/querytask.c
+++ b/source/libs/executor/src/querytask.c
@@ -109,8 +109,8 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand
void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo) {
taosMemoryFreeClear(pSchemaInfo->dbname);
taosMemoryFreeClear(pSchemaInfo->tablename);
- tDeleteSSchemaWrapper(pSchemaInfo->sw);
- tDeleteSSchemaWrapper(pSchemaInfo->qsw);
+ tDeleteSchemaWrapper(pSchemaInfo->sw);
+ tDeleteSchemaWrapper(pSchemaInfo->qsw);
}
int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, SExecTaskInfo* pTaskInfo) {
@@ -197,7 +197,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) {
return pqSw;
}
-static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSSchemaWrapper(pStreamInfo->schema); }
+static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSchemaWrapper(pStreamInfo->schema); }
static void freeBlock(void* pParam) {
SSDataBlock* pBlock = *(SSDataBlock**)pParam;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 130cca9cbb..7cb3c00c1a 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1623,7 +1623,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
pInfo->pRes->info.dataLoad = 1;
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
- blockDataFreeRes((SSDataBlock*)pBlock);
+// blockDataFreeRes((SSDataBlock*)pBlock);
calBlockTbName(pInfo, pInfo->pRes);
return 0;
@@ -1636,8 +1636,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
qDebug("start to exec queue scan, %s", id);
+#if 0
if (pTaskInfo->streamInfo.submit.msgStr != NULL) {
- if (pInfo->tqReader->msg2.msgStr == NULL) {
+ if (pInfo->tqReader->msg.msgStr == NULL) {
SPackedData submit = pTaskInfo->streamInfo.submit;
if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) {
qError("submit msg messed up when initing stream submit block %p", submit.msgStr);
@@ -1649,24 +1650,23 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
while (tqNextBlockImpl(pInfo->tqReader)) {
- SSDataBlock block = {0};
-
- int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader, NULL);
- if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) {
+ int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL);
+ if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) {
continue;
}
- setBlockIntoRes(pInfo, &block, true);
+ setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true);
if (pBlockInfo->rows > 0) {
return pInfo->pRes;
}
}
- pInfo->tqReader->msg2 = (SPackedData){0};
+ pInfo->tqReader->msg = (SPackedData){0};
pTaskInfo->streamInfo.submit = (SPackedData){0};
return NULL;
}
+#endif
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
@@ -1684,25 +1684,25 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) {
return NULL;
}
+
tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pTaskInfo->streamInfo.snapshotVer);
}
if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) {
+
while (1) {
- SSDataBlock block = {0};
- int32_t type = tqNextBlock(pInfo->tqReader, &block);
+ int32_t type = tqNextBlockInWal(pInfo->tqReader);
+ SSDataBlock* pRes = pInfo->tqReader->pResBlock;
// curVersion move to next, so currentOffset = curVersion - 1
tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1);
if (type == FETCH_TYPE__DATA) {
- qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, block.info.rows,
+ qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows,
pTaskInfo->streamInfo.currentOffset.version);
blockDataCleanup(pInfo->pRes);
- setBlockIntoRes(pInfo, &block, true);
+ setBlockIntoRes(pInfo, pRes, true);
if (pInfo->pRes->info.rows > 0) {
- qDebug("doQueueScan get data from log %" PRId64 " rows, return, version:%" PRId64, pInfo->pRes->info.rows,
- pTaskInfo->streamInfo.currentOffset.version);
return pInfo->pRes;
}
} else if (type == FETCH_TYPE__NONE) {
@@ -2055,7 +2055,7 @@ FETCH_NEXT_BLOCK:
NEXT_SUBMIT_BLK:
while (1) {
- if (pInfo->tqReader->msg2.msgStr == NULL) {
+ if (pInfo->tqReader->msg.msgStr == NULL) {
if (pInfo->validBlockIndex >= totBlockNum) {
updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
doClearBufferedBlocks(pInfo);
@@ -2075,14 +2075,12 @@ FETCH_NEXT_BLOCK:
blockDataCleanup(pInfo->pRes);
while (tqNextBlockImpl(pInfo->tqReader)) {
- SSDataBlock block = {0};
-
- int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader, NULL);
- if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) {
+ int32_t code = tqRetrieveDataBlock(pInfo->tqReader, NULL);
+ if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) {
continue;
}
- setBlockIntoRes(pInfo, &block, false);
+ setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, false);
if (updateInfoIgnore(pInfo->pUpdateInfo, &pInfo->pRes->info.window, pInfo->pRes->info.id.groupId,
pInfo->pRes->info.version)) {
@@ -2115,7 +2113,6 @@ FETCH_NEXT_BLOCK:
// record the scan action.
pInfo->numOfExec++;
pOperator->resultInfo.totalRows += pBlockInfo->rows;
- // printDataBlock(pInfo->pRes, "stream scan");
qDebug("scan rows: %" PRId64, pBlockInfo->rows);
if (pBlockInfo->rows > 0) {
@@ -2191,7 +2188,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid);
}
qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType);
- tDeleteSSchemaWrapper(mtInfo.schema);
+ tDeleteSchemaWrapper(mtInfo.schema);
return NULL;
} else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) {
SSnapContext* sContext = pInfo->sContext;
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index a8ecd9b0a2..4c019b3e71 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -5572,7 +5572,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
}
int32_t len = sprintf(st + VARSTR_HEADER_SIZE,
- "Total_Blocks=[%d] Total_Size=[%.2f Kb] Average_size=[%.2f Kb] Compression_Ratio=[%.2f %c]",
+ "Total_Blocks=[%d] Total_Size=[%.2f KB] Average_size=[%.2f KB] Compression_Ratio=[%.2f %c]",
pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%');
varDataSetLen(st, len);
diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c
index a3b067b94d..f921094752 100644
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@@ -313,7 +313,7 @@ void insDestroyTableDataCxt(STableDataCxt* pTableCxt) {
insDestroyBoundColInfo(&pTableCxt->boundColsInfo);
taosArrayDestroyEx(pTableCxt->pValues, destroyColVal);
if (pTableCxt->pData) {
- tDestroySSubmitTbData(pTableCxt->pData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitTbData(pTableCxt->pData, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pTableCxt->pData);
}
taosMemoryFree(pTableCxt);
@@ -324,7 +324,7 @@ void insDestroyVgroupDataCxt(SVgroupDataCxt* pVgCxt) {
return;
}
- tDestroySSubmitReq(pVgCxt->pData, TSDB_MSG_FLG_ENCODE);
+ tDestroySubmitReq(pVgCxt->pData, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pVgCxt->pData);
taosMemoryFree(pVgCxt);
}
@@ -499,7 +499,7 @@ static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uin
int32_t code = TSDB_CODE_SUCCESS;
uint32_t len = 0;
void* pBuf = NULL;
- tEncodeSize(tEncodeSSubmitReq2, pReq, len, code);
+ tEncodeSize(tEncodeSubmitReq, pReq, len, code);
if (TSDB_CODE_SUCCESS == code) {
SEncoder encoder;
len += sizeof(SSubmitReq2Msg);
@@ -511,7 +511,7 @@ static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uin
((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len);
((SSubmitReq2Msg*)pBuf)->version = htobe64(1);
tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg));
- code = tEncodeSSubmitReq2(&encoder, pReq);
+ code = tEncodeSubmitReq(&encoder, pReq);
tEncoderClear(&encoder);
}
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index fcdc6418a0..e753c08f9d 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -6118,17 +6118,50 @@ static bool isEventWindowQuery(SSelectStmt* pSelect) {
return NULL != pSelect->pWindow && QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow);
}
+static bool hasJsonTypeProjection(SSelectStmt* pSelect) {
+ SNode* pProj = NULL;
+ FOREACH(pProj, pSelect->pProjectionList) {
+ if (TSDB_DATA_TYPE_JSON == ((SExprNode*)pProj)->resType.type) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static EDealRes hasColumnOrPseudoColumn(SNode* pNode, void* pContext) {
+ if (QUERY_NODE_COLUMN == nodeType(pNode)) {
+ *(bool*)pContext = true;
+ return DEAL_RES_END;
+ }
+ if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)) {
+ *(bool*)pContext = true;
+ return DEAL_RES_END;
+ }
+ return DEAL_RES_CONTINUE;
+}
+
+static int32_t subtableExprHasColumnOrPseudoColumn(SNode* pNode) {
+ bool hasColumn = false;
+ nodesWalkExprPostOrder(pNode, hasColumnOrPseudoColumn, &hasColumn);
+ return hasColumn;
+}
+
static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type ||
!pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList ||
- crossTableWithUdaf(pSelect) || isEventWindowQuery(pSelect)) {
+ crossTableWithUdaf(pSelect) || isEventWindowQuery(pSelect) || hasJsonTypeProjection(pSelect)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
}
if (NULL != pSelect->pSubtable && TSDB_DATA_TYPE_VARCHAR != ((SExprNode*)pSelect->pSubtable)->resType.type) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"SUBTABLE expression must be of VARCHAR type");
}
+ if (NULL != pSelect->pSubtable && 0 == LIST_LENGTH(pSelect->pPartitionByList) && subtableExprHasColumnOrPseudoColumn(pSelect->pSubtable)) {
+ return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+ "SUBTABLE expression must not has column when no partition by clause");
+ }
+
if (NULL == pSelect->pWindow && STREAM_TRIGGER_AT_ONCE != pStmt->pOptions->triggerType) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"The trigger mode of non window query can only be AT_ONCE");
@@ -6666,22 +6699,40 @@ static int32_t createRealTableForGrantTable(SGrantStmt* pStmt, SRealTableNode**
}
static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, SAlterUserReq* pReq) {
- if (NULL == pStmt->pTagCond) {
- return TSDB_CODE_SUCCESS;
- }
+ SRealTableNode* pTable = NULL;
if ('\0' == pStmt->tabName[0] || '*' == pStmt->tabName[0]) {
- return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
- "The With clause can only be used for table level privilege");
+ if (pStmt->pTagCond) {
+ return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
+ "The With clause can only be used for table level privilege");
+ } else {
+ return TSDB_CODE_SUCCESS;
+ }
}
- pCxt->pCurrStmt = (SNode*)pStmt;
- SRealTableNode* pTable = NULL;
int32_t code = createRealTableForGrantTable(pStmt, &pTable);
if (TSDB_CODE_SUCCESS == code) {
SName name;
code = getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name),
&(pTable->pMeta));
+ if (code) {
+ nodesDestroyNode((SNode*)pTable);
+ return code;
+ }
+
+ if (TSDB_SUPER_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType) {
+ nodesDestroyNode((SNode*)pTable);
+ return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR,
+ "Only supertable and normal table can be granted");
+ }
}
+
+ if (TSDB_CODE_SUCCESS == code && NULL == pStmt->pTagCond) {
+ nodesDestroyNode((SNode*)pTable);
+ return TSDB_CODE_SUCCESS;
+ }
+
+ pCxt->pCurrStmt = (SNode*)pStmt;
+
if (TSDB_CODE_SUCCESS == code) {
code = addNamespace(pCxt, pTable);
}
diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp
index b7ca944ebb..6a08193a39 100644
--- a/source/libs/parser/test/parInitialCTest.cpp
+++ b/source/libs/parser/test/parInitialCTest.cpp
@@ -920,6 +920,10 @@ TEST_F(ParserInitialCTest, createStreamSemanticCheck) {
run("CREATE STREAM s1 INTO st1 AS SELECT PERCENTILE(c1, 30) FROM t1 INTERVAL(10S)",
TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC);
+ run("CREATE STREAM s2 INTO st1 AS SELECT ts, to_json('{c1:1}') FROM st1 PARTITION BY TBNAME",
+ TSDB_CODE_PAR_INVALID_STREAM_QUERY);
+ run("CREATE STREAM s3 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tbname)) "
+ "AS SELECT _WSTART wstart, COUNT(*) cnt FROM st1 INTERVAL(10S)", TSDB_CODE_PAR_INVALID_STREAM_QUERY);
}
/*
diff --git a/source/libs/scalar/test/CMakeLists.txt b/source/libs/scalar/test/CMakeLists.txt
index 32f5e098c5..caaf86264c 100644
--- a/source/libs/scalar/test/CMakeLists.txt
+++ b/source/libs/scalar/test/CMakeLists.txt
@@ -1,4 +1,4 @@
enable_testing()
-#add_subdirectory(filter)
+add_subdirectory(filter)
add_subdirectory(scalar)
diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp
index b59e89fe0d..51ee9b6570 100644
--- a/source/libs/scalar/test/filter/filterTests.cpp
+++ b/source/libs/scalar/test/filter/filterTests.cpp
@@ -33,6 +33,7 @@
#include "os.h"
#include "filter.h"
+#include "filterInt.h"
#include "nodes.h"
#include "scalar.h"
#include "stub.h"
@@ -344,6 +345,7 @@ TEST(timerangeTest, greater_and_lower_not_strict) {
nodesDestroyNode(logicNode1);
}
+#if 0
TEST(columnTest, smallint_column_greater_double_value) {
SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL;
int16_t leftv[5] = {1, 2, 3, 4, 5};
@@ -1337,6 +1339,127 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) {
nodesDestroyNode(logicNode1);
blockDataDestroy(src);
}
+#endif
+
+template
+int32_t compareSignedWithUnsigned(SignedT l, UnsignedT r) {
+ if (l < 0) return -1;
+ auto l_uint64 = static_cast(l);
+ auto r_uint64 = static_cast(r);
+ if (l_uint64 < r_uint64) return -1;
+ if (l_uint64 > r_uint64) return 1;
+ return 0;
+}
+
+template
+int32_t compareUnsignedWithSigned(UnsignedT l, SignedT r) {
+ if (r < 0) return 1;
+ auto l_uint64 = static_cast(l);
+ auto r_uint64 = static_cast(r);
+ if (l_uint64 < r_uint64) return -1;
+ if (l_uint64 > r_uint64) return 1;
+ return 0;
+}
+
+template
+void doCompareWithValueRange_SignedWithUnsigned(__compar_fn_t fp) {
+ int32_t signedMin = -10, signedMax = 10;
+ int32_t unsignedMin = 0, unsignedMax = 10;
+ for (SignedT l = signedMin; l <= signedMax; ++l) {
+ for (UnsignedT r = unsignedMin; r <= unsignedMax; ++r) {
+ ASSERT_EQ(fp(&l, &r), compareSignedWithUnsigned(l, r));
+ }
+ }
+}
+
+template
+void doCompareWithValueRange_UnsignedWithSigned(__compar_fn_t fp) {
+ int32_t signedMin = -10, signedMax = 10;
+ int32_t unsignedMin = 0, unsignedMax = 10;
+ for (UnsignedT l = unsignedMin; l <= unsignedMax; ++l) {
+ for (SignedT r = signedMin; r <= signedMax; ++r) {
+ ASSERT_EQ(fp(&l, &r), compareUnsignedWithSigned(l, r));
+ }
+ }
+}
+
+template
+void doCompareWithValueRange_OnlyLeftType(__compar_fn_t fp, int32_t rType) {
+ switch (rType) {
+ case TSDB_DATA_TYPE_UTINYINT:
+ doCompareWithValueRange_SignedWithUnsigned(fp);
+ break;
+ case TSDB_DATA_TYPE_USMALLINT:
+ doCompareWithValueRange_SignedWithUnsigned(fp);
+ break;
+ case TSDB_DATA_TYPE_UINT:
+ doCompareWithValueRange_SignedWithUnsigned(fp);
+ break;
+ case TSDB_DATA_TYPE_UBIGINT:
+ doCompareWithValueRange_SignedWithUnsigned(fp);
+ break;
+ case TSDB_DATA_TYPE_TINYINT:
+ doCompareWithValueRange_UnsignedWithSigned(fp);
+ break;
+ case TSDB_DATA_TYPE_SMALLINT:
+ doCompareWithValueRange_UnsignedWithSigned(fp);
+ break;
+ case TSDB_DATA_TYPE_INT:
+ doCompareWithValueRange_UnsignedWithSigned(fp);
+ break;
+ case TSDB_DATA_TYPE_BIGINT:
+ doCompareWithValueRange_UnsignedWithSigned(fp);
+ break;
+ default:
+ FAIL();
+ }
+}
+
+void doCompare(const std::vector &lTypes, const std::vector &rTypes, int32_t oper) {
+ for (int i = 0; i < lTypes.size(); ++i) {
+ for (int j = 0; j < rTypes.size(); ++j) {
+ auto fp = filterGetCompFuncEx(lTypes[i], rTypes[j], oper);
+ switch (lTypes[i]) {
+ case TSDB_DATA_TYPE_TINYINT:
+ doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]);
+ break;
+ case TSDB_DATA_TYPE_SMALLINT:
+ doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]);
+ break;
+ case TSDB_DATA_TYPE_INT:
+ doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]);
+ break;
+ case TSDB_DATA_TYPE_BIGINT:
+ doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]);
+ break;
+ case TSDB_DATA_TYPE_UTINYINT:
+ doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]);
+ break;
+ case TSDB_DATA_TYPE_USMALLINT:
+ doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]);
+ break;
+ case TSDB_DATA_TYPE_UINT:
+ doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]);
+ break;
+ case TSDB_DATA_TYPE_UBIGINT:
+ doCompareWithValueRange_OnlyLeftType(fp, rTypes[j]);
+ break;
+ default:
+ FAIL();
+ }
+ }
+ }
+}
+
+TEST(dataCompareTest, signed_and_unsigned_int) {
+ std::vector lType = {TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT,
+ TSDB_DATA_TYPE_BIGINT};
+ std::vector rType = {TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_USMALLINT, TSDB_DATA_TYPE_UINT,
+ TSDB_DATA_TYPE_UBIGINT};
+
+ doCompare(lType, rType, OP_TYPE_GREATER_THAN);
+ doCompare(rType, lType, OP_TYPE_GREATER_THAN);
+}
int main(int argc, char **argv) {
taosSeedRand(taosGetTimestampSec());
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index f33e126068..e10562f5cb 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -250,10 +250,11 @@ int32_t streamExecForAll(SStreamTask* pTask) {
void* pInput = NULL;
// merge multiple input data if possible in the input queue.
+ qDebug("s-task:%s start to extract data block from inputQ", pTask->id.idStr);
+
while (1) {
SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue);
if (qItem == NULL) {
-// qDebug("s-task:%s extract data from input queue, queue is empty, abort", pTask->id.idStr);
break;
}
@@ -298,7 +299,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
}
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
- qDebug("s-task:%s exec begin, numOfBlocks:%d", pTask->id.idStr, batchSize);
+ qDebug("s-task:%s start to execute, numOfBlocks:%d", pTask->id.idStr, batchSize);
streamTaskExecImpl(pTask, pInput, pRes);
@@ -313,6 +314,7 @@ int32_t streamExecForAll(SStreamTask* pTask) {
pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer};
taosWLockLatch(&pTask->pMeta->lock);
+
streamMetaSaveTask(pTask->pMeta, pTask);
if (streamMetaCommit(pTask->pMeta) < 0) {
taosWUnLockLatch(&pTask->pMeta->lock);
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 822ae2a485..af54904c43 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -57,6 +57,13 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
goto _err;
}
+ // task list
+ pMeta->pTaskList = taosArrayInit(4, sizeof(int32_t));
+ if (pMeta->pTaskList == NULL) {
+ terrno = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+
if (streamMetaBegin(pMeta) < 0) {
goto _err;
}
@@ -70,6 +77,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
_err:
taosMemoryFree(pMeta->path);
if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks);
+ if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList);
if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb);
if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb);
if (pMeta->db) tdbClose(pMeta->db);
@@ -100,6 +108,7 @@ void streamMetaClose(SStreamMeta* pMeta) {
}
taosHashCleanup(pMeta->pTasks);
+ pMeta->pTaskList = taosArrayDestroy(pMeta->pTaskList);
taosMemoryFree(pMeta->path);
taosMemoryFree(pMeta);
}
@@ -179,12 +188,21 @@ int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask*
return -1;
}
+ if (streamMetaCommit(pMeta) < 0) {
+ tFreeStreamTask(pTask);
+ return -1;
+ }
+
taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, POINTER_BYTES);
+ taosArrayPush(pMeta->pTaskList, &pTask->id.taskId);
return 0;
}
int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta) {
- return (int32_t) taosHashGetSize(pMeta->pTasks);
+ size_t size = taosHashGetSize(pMeta->pTasks);
+ ASSERT(taosArrayGetSize(pMeta->pTaskList) == taosHashGetSize(pMeta->pTasks));
+
+ return (int32_t) size;
}
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) {
@@ -216,12 +234,23 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t));
if (ppTask) {
SStreamTask* pTask = *ppTask;
+
+ taosWLockLatch(&pMeta->lock);
+
taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t));
tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn);
atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__STOP);
- taosWLockLatch(&pMeta->lock);
+ int32_t num = taosArrayGetSize(pMeta->pTaskList);
+ for(int32_t i = 0; i < num; ++i) {
+ int32_t* pTaskId = taosArrayGet(pMeta->pTaskList, i);
+ if (*pTaskId == taskId) {
+ taosArrayRemove(pMeta->pTaskList, i);
+ break;
+ }
+ }
+
streamMetaReleaseTask(pMeta, pTask);
taosWUnLockLatch(&pMeta->lock);
}
@@ -306,6 +335,8 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
return -1;
}
+ taosArrayPush(pMeta->pTaskList, &pTask->id.taskId);
+
if (pTask->fillHistory) {
pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
streamTaskCheckDownstream(pTask, ver);
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 67c60008fd..f301d9d517 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -193,7 +193,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree);
if (pTask->outputType == TASK_OUTPUT__TABLE) {
- tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper);
+ tDeleteSchemaWrapper(pTask->tbSink.pSchemaWrapper);
taosMemoryFree(pTask->tbSink.pTSchema);
}
diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c
index f9f14c2e00..3506d477d3 100644
--- a/source/libs/sync/src/syncRespMgr.c
+++ b/source/libs/sync/src/syncRespMgr.c
@@ -171,6 +171,8 @@ static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) {
}
void syncRespCleanRsp(SSyncRespMgr *pObj) {
+ if (pObj == NULL) return;
+
SSyncNode *pNode = pObj->data;
sTrace("vgId:%d, clean all resp", pNode->vgId);
diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 6df2b40000..c49b5726b6 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -1814,6 +1814,11 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
*ppVal = pVal;
*vLen = cd.vLen;
+ } else {
+ if (TDB_CELLDECODER_FREE_VAL(&cd)) {
+ tdbTrace("tdb/btree-next2 decoder: %p pVal free: %p", &cd, cd.pVal);
+ tdbFree(cd.pVal);
+ }
}
ret = tdbBtcMoveToNext(pBtc);
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index c23d6d0a1f..ea35f1cfe5 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -587,12 +587,12 @@ void* destroyConnPool(SCliThrd* pThrd) {
static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) {
void* pool = pThrd->pool;
- SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
+ SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1);
STrans* pTranInst = pThrd->pTransInst;
if (plist == NULL) {
SConnList list = {0};
- taosHashPut((SHashObj*)pool, key, strlen(key), (void*)&list, sizeof(list));
- plist = taosHashGet(pool, key, strlen(key));
+ taosHashPut((SHashObj*)pool, key, strlen(key) + 1, (void*)&list, sizeof(list));
+ plist = taosHashGet(pool, key, strlen(key) + 1);
SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList));
QUEUE_INIT(&nList->msgQ);
@@ -627,11 +627,11 @@ static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) {
static SCliConn* getConnFromPool2(SCliThrd* pThrd, char* key, SCliMsg** pMsg) {
void* pool = pThrd->pool;
STrans* pTransInst = pThrd->pTransInst;
- SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
+ SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1);
if (plist == NULL) {
SConnList list = {0};
- taosHashPut((SHashObj*)pool, key, strlen(key), (void*)&list, sizeof(list));
- plist = taosHashGet(pool, key, strlen(key));
+ taosHashPut((SHashObj*)pool, key, strlen(key) + 1, (void*)&list, sizeof(list));
+ plist = taosHashGet(pool, key, strlen(key) + 1);
SMsgList* nList = taosMemoryCalloc(1, sizeof(SMsgList));
QUEUE_INIT(&nList->msgQ);
@@ -717,7 +717,7 @@ static void addConnToPool(void* pool, SCliConn* conn) {
cliDestroyConnMsgs(conn, false);
if (conn->list == NULL) {
- conn->list = taosHashGet((SHashObj*)pool, conn->ip, strlen(conn->ip));
+ conn->list = taosHashGet((SHashObj*)pool, conn->ip, strlen(conn->ip) + 1);
}
SConnList* pList = conn->list;
@@ -822,7 +822,8 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
return;
}
if (nread < 0) {
- tWarn("%s conn %p read error:%s, ref:%d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), T_REF_VAL_GET(conn));
+ tDebug("%s conn %p read error:%s, ref:%d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread),
+ T_REF_VAL_GET(conn));
conn->broken = true;
cliHandleExcept(conn);
}
@@ -875,8 +876,8 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
connList->list->numOfConn--;
connList->size--;
} else {
- SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->ip, strlen(conn->ip));
- connList->list->numOfConn--;
+ SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->ip, strlen(conn->ip) + 1);
+ if (connList != NULL) connList->list->numOfConn--;
}
conn->list = NULL;
pThrd->newConnCount--;
@@ -1269,7 +1270,7 @@ static void cliHandleFastFail(SCliConn* pConn, int status) {
if (pMsg != NULL && REQUEST_NO_RESP(&pMsg->msg) &&
(pTransInst->failFastFp != NULL && pTransInst->failFastFp(pMsg->msg.msgType))) {
- SFailFastItem* item = taosHashGet(pThrd->failFastCache, pConn->ip, strlen(pConn->ip));
+ SFailFastItem* item = taosHashGet(pThrd->failFastCache, pConn->ip, strlen(pConn->ip) + 1);
int64_t cTimestamp = taosGetTimestampMs();
if (item != NULL) {
int32_t elapse = cTimestamp - item->timestamp;
@@ -1281,7 +1282,7 @@ static void cliHandleFastFail(SCliConn* pConn, int status) {
}
} else {
SFailFastItem item = {.count = 1, .timestamp = cTimestamp};
- taosHashPut(pThrd->failFastCache, pConn->ip, strlen(pConn->ip), &item, sizeof(SFailFastItem));
+ taosHashPut(pThrd->failFastCache, pConn->ip, strlen(pConn->ip) + 1, &item, sizeof(SFailFastItem));
}
}
} else {
@@ -1459,7 +1460,7 @@ FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) {
}
static FORCE_INLINE uint32_t cliGetIpFromFqdnCache(SHashObj* cache, char* fqdn) {
uint32_t addr = 0;
- uint32_t* v = taosHashGet(cache, fqdn, strlen(fqdn));
+ uint32_t* v = taosHashGet(cache, fqdn, strlen(fqdn) + 1);
if (v == NULL) {
addr = taosGetIpv4FromFqdn(fqdn);
if (addr == 0xffffffff) {
@@ -1468,7 +1469,7 @@ static FORCE_INLINE uint32_t cliGetIpFromFqdnCache(SHashObj* cache, char* fqdn)
return addr;
}
- taosHashPut(cache, fqdn, strlen(fqdn), &addr, sizeof(addr));
+ taosHashPut(cache, fqdn, strlen(fqdn) + 1, &addr, sizeof(addr));
} else {
addr = *v;
}
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 28fb474972..269c7ecf9b 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -314,7 +314,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
return;
}
- tWarn("%s conn %p read error:%s", transLabel(pTransInst), conn, uv_err_name(nread));
+ tDebug("%s conn %p read error:%s", transLabel(pTransInst), conn, uv_err_name(nread));
if (nread < 0) {
conn->broken = true;
if (conn->status == ConnAcquire) {
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 6154e30938..b29e36efdc 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -37,7 +37,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) {
if (cond) {
pReader->cond = *cond;
} else {
- pReader->cond.scanUncommited = 0;
+// pReader->cond.scanUncommited = 0;
pReader->cond.scanNotApplied = 0;
pReader->cond.scanMeta = 0;
pReader->cond.enableRef = 0;
@@ -74,13 +74,18 @@ int32_t walNextValidMsg(SWalReader *pReader) {
int64_t lastVer = walGetLastVer(pReader->pWal);
int64_t committedVer = walGetCommittedVer(pReader->pWal);
int64_t appliedVer = walGetAppliedVer(pReader->pWal);
- int64_t endVer = pReader->cond.scanUncommited ? lastVer : committedVer;
- endVer = TMIN(appliedVer, endVer);
+ while(appliedVer < committedVer){ // wait apply ver equal to commit ver, otherwise may lost data when consume data [TD-24010]
+ wDebug("vgId:%d, wal apply ver:%"PRId64" smaller than commit ver:%"PRId64", so sleep 1ms", pReader->pWal->cfg.vgId, appliedVer, committedVer);
+ taosMsleep(1);
+ appliedVer = walGetAppliedVer(pReader->pWal);
+ }
+// int64_t endVer = pReader->cond.scanUncommited ? lastVer : committedVer;
+// endVer = TMIN(appliedVer, endVer);
wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64
- ", applied index:%" PRId64 ", end index:%" PRId64,
- pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer);
- while (fetchVer <= endVer) {
+ ", applied index:%" PRId64,
+ pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer);
+ while (fetchVer <= committedVer) {
if (walFetchHeadNew(pReader, fetchVer) < 0) {
return -1;
}
@@ -237,6 +242,7 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
}
seeked = true;
}
+
while (1) {
contLen = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead));
if (contLen == sizeof(SWalCkHead)) {
diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c
index 129faaacc8..136b8cf022 100644
--- a/source/os/src/osLocale.c
+++ b/source/os/src/osLocale.c
@@ -171,7 +171,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) {
strcpy(outLocale, "en_US.UTF-8");
} else {
tstrncpy(outLocale, locale, TD_LOCALE_LEN);
- printf("locale not configured, set to system default:%s\n", outLocale);
+ //printf("locale not configured, set to system default:%s\n", outLocale);
}
// if user does not specify the charset, extract it from locale
diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c
index f8f78ae6a5..dc57ed97b2 100644
--- a/source/util/src/tcompare.c
+++ b/source/util/src/tcompare.c
@@ -308,17 +308,19 @@ int32_t compareInt8Uint16(const void *pLeft, const void *pRight) {
int32_t compareInt8Uint32(const void *pLeft, const void *pRight) {
int8_t left = GET_INT8_VAL(pLeft);
+ if (left < 0) return -1;
uint32_t right = GET_UINT32_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if ((uint32_t)left > right) return 1;
+ if ((uint32_t)left < right) return -1;
return 0;
}
int32_t compareInt8Uint64(const void *pLeft, const void *pRight) {
int8_t left = GET_INT8_VAL(pLeft);
+ if (left < 0) return -1;
uint64_t right = GET_UINT64_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if ((uint64_t)left > right) return 1;
+ if ((uint64_t)left < right) return -1;
return 0;
}
@@ -380,17 +382,19 @@ int32_t compareInt16Uint16(const void *pLeft, const void *pRight) {
int32_t compareInt16Uint32(const void *pLeft, const void *pRight) {
int16_t left = GET_INT16_VAL(pLeft);
+ if (left < 0) return -1;
uint32_t right = GET_UINT32_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if ((uint32_t)left > right) return 1;
+ if ((uint32_t)left < right) return -1;
return 0;
}
int32_t compareInt16Uint64(const void *pLeft, const void *pRight) {
int16_t left = GET_INT16_VAL(pLeft);
+ if (left < 0) return -1;
uint64_t right = GET_UINT64_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if ((uint64_t)left > right) return 1;
+ if ((uint64_t)left < right) return -1;
return 0;
}
@@ -452,17 +456,19 @@ int32_t compareInt32Uint16(const void *pLeft, const void *pRight) {
int32_t compareInt32Uint32(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft);
+ if (left < 0) return -1;
uint32_t right = GET_UINT32_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if ((uint32_t)left > right) return 1;
+ if ((uint32_t)left < right) return -1;
return 0;
}
int32_t compareInt32Uint64(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft);
+ if (left < 0) return -1;
uint64_t right = GET_UINT64_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if ((uint64_t)left > right) return 1;
+ if ((uint64_t)left < right) return -1;
return 0;
}
@@ -532,9 +538,10 @@ int32_t compareInt64Uint32(const void *pLeft, const void *pRight) {
int32_t compareInt64Uint64(const void *pLeft, const void *pRight) {
int64_t left = GET_INT64_VAL(pLeft);
+ if (left < 0) return -1;
uint64_t right = GET_UINT64_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if ((uint64_t)left > right) return 1;
+ if ((uint64_t)left < right) return -1;
return 0;
}
@@ -857,24 +864,27 @@ int32_t compareUint16Uint64(const void *pLeft, const void *pRight) {
int32_t compareUint32Int8(const void *pLeft, const void *pRight) {
uint32_t left = GET_UINT32_VAL(pLeft);
int8_t right = GET_INT8_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if (right < 0) return 1;
+ if (left > (uint32_t)right) return 1;
+ if (left < (uint32_t)right) return -1;
return 0;
}
int32_t compareUint32Int16(const void *pLeft, const void *pRight) {
uint32_t left = GET_UINT32_VAL(pLeft);
int16_t right = GET_INT16_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if (right < 0) return 1;
+ if (left > (uint32_t)right) return 1;
+ if (left < (uint32_t)right) return -1;
return 0;
}
int32_t compareUint32Int32(const void *pLeft, const void *pRight) {
uint32_t left = GET_UINT32_VAL(pLeft);
int32_t right = GET_INT32_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if (right < 0) return 1;
+ if (left > (uint32_t)right) return 1;
+ if (left < (uint32_t)right) return -1;
return 0;
}
@@ -929,32 +939,36 @@ int32_t compareUint32Uint64(const void *pLeft, const void *pRight) {
int32_t compareUint64Int8(const void *pLeft, const void *pRight) {
uint64_t left = GET_UINT64_VAL(pLeft);
int8_t right = GET_INT8_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if (right < 0) return 1;
+ if (left > (uint64_t)right) return 1;
+ if (left < (uint64_t)right) return -1;
return 0;
}
int32_t compareUint64Int16(const void *pLeft, const void *pRight) {
uint64_t left = GET_UINT64_VAL(pLeft);
int16_t right = GET_INT16_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if (right < 0) return 1;
+ if (left > (uint64_t)right) return 1;
+ if (left < (uint64_t)right) return -1;
return 0;
}
int32_t compareUint64Int32(const void *pLeft, const void *pRight) {
uint64_t left = GET_UINT64_VAL(pLeft);
int32_t right = GET_INT32_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if (right < 0) return 1;
+ if (left > (uint64_t)right) return 1;
+ if (left < (uint64_t)right) return -1;
return 0;
}
int32_t compareUint64Int64(const void *pLeft, const void *pRight) {
uint64_t left = GET_UINT64_VAL(pLeft);
int64_t right = GET_INT64_VAL(pRight);
- if (left > right) return 1;
- if (left < right) return -1;
+ if (right < 0) return 1;
+ if (left > (uint64_t)right) return 1;
+ if (left < (uint64_t)right) return -1;
return 0;
}
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 002d605793..6f726c7eff 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -122,7 +122,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, "No write permission")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, "Connection killed")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, "Syntax error in SQL")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DB_NOT_SELECTED, "Database not specified or available")
-TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME, "Table does not exist")
+//TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME, "Table does not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too long")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line")
@@ -203,6 +203,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists
TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION, "Invalid stable options")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ROW_BYTES, "Invalid row bytes")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_VALUE_OVERFLOW, "out of range and overflow")
// mnode-func
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name")
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 7e58ea9a13..4065ac5bee 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -136,6 +136,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sysinfo.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_manage.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel.py
,,n,system-test,python3 ./test.py -f 0-others/compatibility.py
diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh
index ffc23ba6d2..2d736e1414 100755
--- a/tests/parallel_test/run_case.sh
+++ b/tests/parallel_test/run_case.sh
@@ -79,7 +79,7 @@ md5sum /home/TDinternal/debug/build/lib/libtaos.so
#define taospy 2.7.6
pip3 list|grep taospy
pip3 uninstall taospy -y
-pip3 install taospy==2.7.6
+pip3 install --default-timeout=120 taospy==2.7.6
$TIMEOUT_CMD $cmd
RET=$?
diff --git a/tests/pytest/auto_crash_gen.py b/tests/pytest/auto_crash_gen.py
index 56629ede13..00e1786399 100755
--- a/tests/pytest/auto_crash_gen.py
+++ b/tests/pytest/auto_crash_gen.py
@@ -1,3 +1,4 @@
+import datetime
import os
import socket
import requests
@@ -238,17 +239,7 @@ def start_taosd():
start_cmd = 'cd %s && python3 test.py >>/dev/null '%(start_path)
os.system(start_cmd)
-def get_cmds(args_list):
- # build_path = get_path()
- # if repo == "community":
- # crash_gen_path = build_path[:-5]+"community/tests/pytest/"
- # elif repo == "TDengine":
- # crash_gen_path = build_path[:-5]+"/tests/pytest/"
- # else:
- # pass
-
- # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path)
-
+def get_cmds(args_list):
crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode)
return crash_gen_cmd
@@ -295,7 +286,7 @@ def check_status():
elif "Crash_Gen is now exiting with status code: 0" in run_code:
return 0
else:
- return 2
+ return 2
def main():
@@ -310,7 +301,7 @@ def main():
build_path = get_path()
- os.system("pip install git+https://github.com/taosdata/taos-connector-python.git")
+
if repo =="community":
crash_gen_path = build_path[:-5]+"community/tests/pytest/"
elif repo =="TDengine":
@@ -334,7 +325,9 @@ def main():
if not os.path.exists(run_dir):
os.mkdir(run_dir)
print(crash_cmds)
+ starttime = datetime.datetime.now()
run_crash_gen(crash_cmds)
+ endtime = datetime.datetime.now()
status = check_status()
print("exit status : ", status)
@@ -349,7 +342,29 @@ def main():
print('======== crash_gen run sucess and exit as expected ========')
try:
- text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}"
+ cmd = crash_cmds.split('&')[2]
+ if status == 0:
+ log_dir = "none"
+ else:
+ log_dir= "/root/pxiao/crash_gen_logs"
+
+ if status == 3:
+ core_dir = "/root/pxiao/crash_gen_logs"
+ else:
+ core_dir = "none"
+
+ text = f'''
+ exit status: {msg_dict[status]}
+ test scope: crash_gen
+ owner: pxiao
+ hostname: {hostname}
+ start time: {starttime}
+ end time: {endtime}
+ git commit : {git_commit}
+ log dir: {log_dir}
+ core dir: {core_dir}
+ cmd: {cmd}'''
+
send_msg(get_msg(text))
except Exception as e:
print("exception:", e)
diff --git a/tests/pytest/auto_crash_gen_valgrind.py b/tests/pytest/auto_crash_gen_valgrind.py
index 22fc5a480f..e37cda0a27 100755
--- a/tests/pytest/auto_crash_gen_valgrind.py
+++ b/tests/pytest/auto_crash_gen_valgrind.py
@@ -1,6 +1,7 @@
#!/usr/bin/python3
+import datetime
import os
import socket
import requests
@@ -241,15 +242,6 @@ def start_taosd():
os.system(start_cmd +">>/dev/null")
def get_cmds(args_list):
- # build_path = get_path()
- # if repo == "community":
- # crash_gen_path = build_path[:-5]+"community/tests/pytest/"
- # elif repo == "TDengine":
- # crash_gen_path = build_path[:-5]+"/tests/pytest/"
- # else:
- # pass
-
- # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path)
crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode)
return crash_gen_cmd
@@ -343,7 +335,6 @@ def main():
args = limits(args)
build_path = get_path()
- os.system("pip install git+https://github.com/taosdata/taos-connector-python.git >>/dev/null")
if repo =="community":
crash_gen_path = build_path[:-5]+"community/tests/pytest/"
elif repo =="TDengine":
@@ -368,7 +359,9 @@ def main():
if not os.path.exists(run_dir):
os.mkdir(run_dir)
print(crash_cmds)
+ starttime = datetime.datetime.now()
run_crash_gen(crash_cmds)
+ endtime = datetime.datetime.now()
status = check_status()
# back_path = os.path.join(core_path,"valgrind_report")
@@ -384,8 +377,30 @@ def main():
print('======== crash_gen run sucess and exit as expected ========')
try:
- text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}"
- send_msg(get_msg(text))
+ cmd = crash_cmds.split('&')[2]
+ if status == 0:
+ log_dir = "none"
+ else:
+ log_dir= "/root/pxiao/crash_gen_logs"
+
+ if status == 3:
+ core_dir = "/root/pxiao/crash_gen_logs"
+ else:
+ core_dir = "none"
+
+ text = f'''
+ exit status: {msg_dict[status]}
+ test scope: crash_gen
+ owner: pxiao
+ hostname: {hostname}
+ start time: {starttime}
+ end time: {endtime}
+ git commit : {git_commit}
+ log dir: {log_dir}
+ core dir: {core_dir}
+ cmd: {cmd}'''
+
+ send_msg(get_msg(text))
except Exception as e:
print("exception:", e)
exit(status)
diff --git a/tests/pytest/auto_crash_gen_valgrind_cluster.py b/tests/pytest/auto_crash_gen_valgrind_cluster.py
index 547de9af47..af19836a83 100755
--- a/tests/pytest/auto_crash_gen_valgrind_cluster.py
+++ b/tests/pytest/auto_crash_gen_valgrind_cluster.py
@@ -1,6 +1,7 @@
#!/usr/bin/python3
+import datetime
import os
import socket
import requests
@@ -241,16 +242,7 @@ def start_taosd():
os.system(start_cmd +">>/dev/null")
def get_cmds(args_list):
- # build_path = get_path()
- # if repo == "community":
- # crash_gen_path = build_path[:-5]+"community/tests/pytest/"
- # elif repo == "TDengine":
- # crash_gen_path = build_path[:-5]+"/tests/pytest/"
- # else:
- # pass
-
- # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path)
-
+
crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode)
return crash_gen_cmd
@@ -342,8 +334,7 @@ def main():
args = random_args(args_list)
args = limits(args)
- build_path = get_path()
- os.system("pip install git+https://github.com/taosdata/taos-connector-python.git >>/dev/null")
+ build_path = get_path()
if repo =="community":
crash_gen_path = build_path[:-5]+"community/tests/pytest/"
elif repo =="TDengine":
@@ -368,7 +359,9 @@ def main():
if not os.path.exists(run_dir):
os.mkdir(run_dir)
print(crash_cmds)
+ starttime = datetime.datetime.now()
run_crash_gen(crash_cmds)
+ endtime = datetime.datetime.now()
status = check_status()
# back_path = os.path.join(core_path,"valgrind_report")
@@ -384,7 +377,29 @@ def main():
print('======== crash_gen run sucess and exit as expected ========')
try:
- text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}"
+ cmd = crash_cmds.split('&')[2]
+ if status == 0:
+ log_dir = "none"
+ else:
+ log_dir= "/root/pxiao/crash_gen_logs"
+
+ if status == 3:
+ core_dir = "/root/pxiao/crash_gen_logs"
+ else:
+ core_dir = "none"
+
+ text = f'''
+ exit status: {msg_dict[status]}
+ test scope: crash_gen
+ owner: pxiao
+ hostname: {hostname}
+ start time: {starttime}
+ end time: {endtime}
+ git commit : {git_commit}
+ log dir: {log_dir}
+ core dir: {core_dir}
+ cmd: {cmd}'''
+
send_msg(get_msg(text))
except Exception as e:
print("exception:", e)
diff --git a/tests/script/tsim/db/error1.sim b/tests/script/tsim/db/error1.sim
index 32dbe826cc..64b17125aa 100644
--- a/tests/script/tsim/db/error1.sim
+++ b/tests/script/tsim/db/error1.sim
@@ -58,16 +58,16 @@ if $data23 != 0 then
return -1
endi
-print ========== stop dnode2
-system sh/exec.sh -n dnode2 -s stop -x SIGKILL
+#print ========== stop dnode2
+#system sh/exec.sh -n dnode2 -s stop -x SIGKILL
-sleep 1000
-print =============== drop database
-sql_error drop database d1
+#sleep 1000
+#print =============== drop database
+sql drop database d1
-print ========== start dnode2
-system sh/exec.sh -n dnode2 -s start
-sleep 1000
+#print ========== start dnode2
+#system sh/exec.sh -n dnode2 -s start
+#sleep 1000
print =============== re-create database
$x = 0
diff --git a/tests/system-test/0-others/user_privilege.py b/tests/system-test/0-others/user_privilege.py
new file mode 100644
index 0000000000..6d49ebfbfe
--- /dev/null
+++ b/tests/system-test/0-others/user_privilege.py
@@ -0,0 +1,120 @@
+###################################################################
+# Copyright (c) 2016 by TAOS Technologies, Inc.
+# All rights reserved.
+#
+# This file is proprietary and confidential to TAOS Technologies.
+# No part of this file may be reproduced, stored, transmitted,
+# disclosed or used in any form or by any means other than as
+# expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import taos
+from taos.tmq import *
+from util.cases import *
+from util.common import *
+from util.log import *
+from util.sql import *
+from util.sqlset import *
+
+
+class TDTestCase:
+ def init(self, conn, logSql, replicaVar=1):
+ self.replicaVar = int(replicaVar)
+ tdLog.debug("start to execute %s" % __file__)
+ tdSql.init(conn.cursor())
+ self.setsql = TDSetSql()
+ self.stbname = 'stb'
+ self.binary_length = 20 # the length of binary for column_dict
+ self.nchar_length = 20 # the length of nchar for column_dict
+ self.column_dict = {
+ 'ts': 'timestamp',
+ 'col1': 'float',
+ 'col2': 'int',
+ 'col3': 'float',
+ }
+
+ self.tag_dict = {
+ 't1': 'int',
+ 't2': f'binary({self.binary_length})'
+ }
+
+ self.tag_list = [
+ f'1, "Beijing"',
+ f'2, "Shanghai"',
+ f'3, "Guangzhou"',
+ f'4, "Shenzhen"'
+ ]
+
+ self.values_list = [
+ f'now, 9.1, 200, 0.3'
+ ]
+
+ self.tbnum = 4
+
+ def create_user(self):
+ user_name = 'test'
+ tdSql.execute(f'create user {user_name} pass "test"')
+ tdSql.execute(f'grant read on db.stb with t2 = "Beijing" to {user_name}')
+
+ def prepare_data(self):
+ tdSql.execute(self.setsql.set_create_stable_sql(self.stbname, self.column_dict, self.tag_dict))
+ for i in range(self.tbnum):
+ tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})')
+ for j in self.values_list:
+ tdSql.execute(f'insert into {self.stbname}_{i} values({j})')
+
+ def user_privilege_check(self):
+ testconn = taos.connect(user='test', password='test')
+ expectErrNotOccured = False
+
+ try:
+ sql = "select count(*) from db.stb where t2 = 'Beijing'"
+ res = testconn.query(sql)
+ data = res.fetch_all()
+ count = data[0][0]
+ except BaseException:
+ expectErrNotOccured = True
+
+ if expectErrNotOccured:
+ caller = inspect.getframeinfo(inspect.stack()[1][0])
+ tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, expect error not occured")
+ elif count != 1:
+ tdLog.exit(f"{sql}, expect result doesn't match")
+ pass
+
+ def user_privilege_error_check(self):
+ testconn = taos.connect(user='test', password='test')
+ expectErrNotOccured = False
+
+ sql_list = ["alter talbe db.stb_1 set t2 = 'Wuhan'", "drop table db.stb_1"]
+
+ for sql in sql_list:
+ try:
+ res = testconn.execute(sql)
+ except BaseException:
+ expectErrNotOccured = True
+
+ if expectErrNotOccured:
+ pass
+ else:
+ caller = inspect.getframeinfo(inspect.stack()[1][0])
+ tdLog.exit(f"{caller.filename}({caller.lineno}) failed: sql:{sql}, expect error not occured")
+ pass
+
+ def run(self):
+ tdSql.prepare()
+ self.prepare_data()
+ self.create_user()
+ self.user_privilege_check()
+ self.user_privilege_error_check()
+
+ def stop(self):
+ tdSql.close()
+ tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py
index f96ed8a3ff..2f97118fbf 100644
--- a/tests/system-test/2-query/sml.py
+++ b/tests/system-test/2-query/sml.py
@@ -34,6 +34,9 @@ class TDTestCase:
if ret != 0:
tdLog.info("sml_test ret != 0")
+ tdSql.query(f"select * from ts3303.stb2")
+ tdSql.query(f"select * from ts3303.meters")
+
# tdSql.execute('use sml_db')
tdSql.query(f"select * from {dbname}.t_b7d815c9222ca64cdf2614c61de8f211")
tdSql.checkRows(1)
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 5ac32eaad9..0f91bdeeda 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -554,7 +554,12 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
if (tsEnableScience) {
printf("%*e", width, GET_FLOAT_VAL(val));
} else {
- printf("%*.5f", width, GET_FLOAT_VAL(val));
+ n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.5f", width, GET_FLOAT_VAL(val));
+ if (n > TMAX(20, width)) {
+ printf("%*e", width, GET_FLOAT_VAL(val));
+ } else {
+ printf("%s", buf);
+ }
}
break;
case TSDB_DATA_TYPE_DOUBLE:
diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c
index f1f4bbc1fd..f1dc8ebe79 100644
--- a/utils/test/c/sml_test.c
+++ b/utils/test/c/sml_test.c
@@ -1159,6 +1159,57 @@ int sml_td23881_Test() {
return code;
}
+int sml_ts3303_Test() {
+ TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+
+ TAOS_RES *pRes = taos_query(taos, "drop database if exists ts3303");
+ taos_free_result(pRes);
+
+ pRes = taos_query(taos, "create database if not exists ts3303");
+ taos_free_result(pRes);
+
+ const char *sql[] = {
+ "stb2,t1=1,dataModelName=t0 f1=283i32 1632299372000",
+ "stb2,t1=1,dataModelName=t0 f1=106i32 1632299378000",
+ "stb2,t1=4,dataModelName=t0 f1=144i32 1629716944000",
+ "stb2,t1=4,dataModelName=t0 f1=125i32 1629717012000",
+ "stb2,t1=4,dataModelName=t0 f1=144i32 1629717012000",
+ "stb2,t1=4,dataModelName=t0 f1=107i32 1629717013000",
+ "stb2,t1=6,dataModelName=t0 f1=154i32 1629717140000",
+ "stb2,t1=6,dataModelName=t0 f1=93i32 1629717140000",
+ "stb2,t1=6,dataModelName=t0 f1=134i32 1629717140000",
+ "stb2,t1=4,dataModelName=t0 f1=73i32 1629717140000",
+ "stb2,t1=4,dataModelName=t0 f1=83i32 1629717140000",
+ "stb2,t1=4,dataModelName=t0 f1=72i32 1629717140000",
+ };
+
+ const char *sql1[] = {
+ "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833339000000",
+ "meters,groupid=2,location=California.LosAngeles current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833339000000",
+ };
+
+ pRes = taos_query(taos, "use ts3303");
+ taos_free_result(pRes);
+
+ pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL,
+ TSDB_SML_TIMESTAMP_MILLI_SECONDS, 20);
+
+ int code = taos_errno(pRes);
+ printf("%s result0:%s\n", __FUNCTION__, taos_errstr(pRes));
+ taos_free_result(pRes);
+ ASSERT(code == 0);
+
+ pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL,
+ TSDB_SML_TIMESTAMP_NANO_SECONDS, 20);
+
+ printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes));
+ taos_free_result(pRes);
+
+ taos_close(taos);
+
+ return code;
+}
+
int sml_ttl_Test() {
TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -1336,6 +1387,9 @@ int main(int argc, char *argv[]) {
ASSERT(!ret);
ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file
ASSERT(!ret);
+ ret = sml_ts3303_Test(); // this test case need config sml table name using ./sml_test config_file
+ ASSERT(!ret);
+
// for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){
// printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i])));
// }